Index: src/backend/commands/copy.c =================================================================== RCS file: /projects/cvsroot/pgsql/src/backend/commands/copy.c,v retrieving revision 1.247 diff -c -r1.247 copy.c *** src/backend/commands/copy.c 10 Jul 2005 21:13:58 -0000 1.247 --- src/backend/commands/copy.c 2 Aug 2005 14:38:37 -0000 *************** *** 50,55 **** --- 50,56 ---- #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7')) #define OCTVALUE(c) ((c) - '0') + #define COPY_BUF_SIZE 65536 /* * Represents the different source/dest cases we need to worry about at *************** *** 63,78 **** } CopyDest; /* - * State indicator showing what stopped CopyReadAttribute() - */ - typedef enum CopyReadResult - { - NORMAL_ATTR, - END_OF_LINE, - UNTERMINATED_FIELD - } CopyReadResult; - - /* * Represents the end-of-line terminator type of the input */ typedef enum EolType --- 64,69 ---- *************** *** 97,102 **** --- 88,96 ---- static EolType eol_type; /* EOL type of input */ static int client_encoding; /* remote side's character encoding */ static int server_encoding; /* local encoding */ + static char eol_ch[2]; /* The byte values of the 1 or 2 eol bytes */ + static bool client_encoding_only; /* true if client encoding is a non + * supported server encoding */ /* these are just for error messages, see copy_in_error_callback */ static bool copy_binary; /* is it a binary copy? */ *************** *** 104,126 **** static int copy_lineno; /* line number for error messages */ static const char *copy_attname; /* current att for error messages */ /* * These static variables are used to avoid incurring overhead for each ! * attribute processed. attribute_buf is reused on each CopyReadAttribute * call to hold the string being read in. Under normal use it will soon * grow to a suitable size, and then we will avoid palloc/pfree overhead * for subsequent attributes. Note that CopyReadAttribute returns a pointer ! * to attribute_buf's data buffer! */ ! static StringInfoData attribute_buf; /* * Similarly, line_buf holds the whole input line being processed (its * cursor field points to the next character to be read by CopyReadAttribute). * The input cycle is first to read the whole line into line_buf, convert it * to server encoding, and then extract individual attribute fields into ! * attribute_buf. (We used to have CopyReadAttribute read the input source * directly, but that caused a lot of encoding issues and unnecessary logic * complexity.) */ --- 98,141 ---- static int copy_lineno; /* line number for error messages */ static const char *copy_attname; /* current att for error messages */ + /* + * Static variables for buffered input parsing + */ + static char input_buf[COPY_BUF_SIZE + 1]; /* extra byte for '\0' */ + static int buffer_index; /* input buffer index */ + static bool end_marker; + static char *begloc; + static char *endloc; + static bool buf_done; /* finished processing the current buffer */ + static bool line_done; /* finished processing the whole line or + * stopped in the middle */ + /* these are for CSV format */ + static bool in_quote; + static bool last_was_esc; + + /* these are for TEXT format */ + static bool esc_in_prevbuf; /* escape was last character of the data + * input buffer */ + static bool cr_in_prevbuf; /* CR was last character of the data input + * buffer */ + /* * These static variables are used to avoid incurring overhead for each ! * attribute processed. attr_buf is reused on each CopyReadAttribute * call to hold the string being read in. Under normal use it will soon * grow to a suitable size, and then we will avoid palloc/pfree overhead * for subsequent attributes. Note that CopyReadAttribute returns a pointer ! * to attr_buf's data buffer! */ ! static StringInfoData attr_buf; /* * Similarly, line_buf holds the whole input line being processed (its * cursor field points to the next character to be read by CopyReadAttribute). * The input cycle is first to read the whole line into line_buf, convert it * to server encoding, and then extract individual attribute fields into ! * attr_buf. (We used to have CopyReadAttribute read the input source * directly, but that caused a lot of encoding issues and unnecessary logic * complexity.) */ *************** *** 137,155 **** static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids, char *delim, char *null_print, bool csv_mode, char *quote, char *escape, List *force_notnull_atts, bool header_line); ! static bool CopyReadLine(char * quote, char * escape); ! static char *CopyReadAttribute(const char *delim, const char *null_print, ! CopyReadResult *result, bool *isnull); ! static char *CopyReadAttributeCSV(const char *delim, const char *null_print, ! char *quote, char *escape, ! CopyReadResult *result, bool *isnull); static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); static void CopyAttributeOut(char *string, char *delim); static void CopyAttributeOutCSV(char *string, char *delim, char *quote, char *escape, bool force_quote); static List *CopyGetAttnums(Relation rel, List *attnamelist); static void limit_printout_length(StringInfo buf); /* Internal communications functions */ static void SendCopyBegin(bool binary, int natts); --- 152,176 ---- static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids, char *delim, char *null_print, bool csv_mode, char *quote, char *escape, List *force_notnull_atts, bool header_line); ! static bool CopyReadLineText(size_t bytesread, char *escape); ! static bool CopyReadLineCSV(size_t bytesread, char *quote, char *escape); ! static void CopyReadAttributesText(const char *delim, const char *escape, const char *null_print, ! int null_print_len, char *nulls, List *attnumlist, ! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr); ! static void CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote, ! char *escape, int null_print_len, char *nulls, List *attnumlist, ! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr); static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); + static char *CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len, + bool *isnull); static void CopyAttributeOut(char *string, char *delim); static void CopyAttributeOutCSV(char *string, char *delim, char *quote, char *escape, bool force_quote); static List *CopyGetAttnums(Relation rel, List *attnamelist); static void limit_printout_length(StringInfo buf); + static bool DetectLineEnd(size_t bytesread, char *quote, char *escape); + /* Internal communications functions */ static void SendCopyBegin(bool binary, int natts); *************** *** 159,175 **** static void CopySendString(const char *str); static void CopySendChar(char c); static void CopySendEndOfRow(bool binary); ! static void CopyGetData(void *databuf, int datasize); ! static int CopyGetChar(void); #define CopyGetEof() (fe_eof) - static int CopyPeekChar(void); - static void CopyDonePeek(int c, bool pickup); static void CopySendInt32(int32 val); static int32 CopyGetInt32(void); static void CopySendInt16(int16 val); static int16 CopyGetInt16(void); /* * Send copy start/stop messages for frontend copies. These have changed --- 180,197 ---- static void CopySendString(const char *str); static void CopySendChar(char c); static void CopySendEndOfRow(bool binary); ! static int CopyGetData(void *databuf, int datasize); #define CopyGetEof() (fe_eof) static void CopySendInt32(int32 val); static int32 CopyGetInt32(void); static void CopySendInt16(int16 val); static int16 CopyGetInt16(void); + /* byte scaning utils */ + static char *scanTextLine(const char *s, char c, size_t len); + static char *scanCSVLine(const char *s, char c1, char c2, char c3, size_t len); + static char *scanTextAttr(const char *s, char c1, char c2, size_t len); /* * Send copy start/stop messages for frontend copies. These have changed *************** *** 382,395 **** * It seems unwise to allow the COPY IN to complete normally in that case. * * NB: no data conversion is applied by these functions */ ! static void CopyGetData(void *databuf, int datasize) { switch (copy_dest) { case COPY_FILE: ! fread(databuf, datasize, 1, copy_file); if (feof(copy_file)) fe_eof = true; break; --- 404,422 ---- * It seems unwise to allow the COPY IN to complete normally in that case. * * NB: no data conversion is applied by these functions + * + * Returns: the number of bytes that were successfully read + * into the data buffer. */ ! static int CopyGetData(void *databuf, int datasize) { + size_t bytesread = 0; + switch (copy_dest) { case COPY_FILE: ! bytesread = fread(databuf, 1, datasize, copy_file); if (feof(copy_file)) fe_eof = true; break; *************** *** 401,406 **** --- 428,435 ---- (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("unexpected EOF on client connection"))); } + bytesread += datasize; /* update the count of bytes that were + * read so far */ break; case COPY_NEW_FE: while (datasize > 0 && !fe_eof) *************** *** 429,435 **** case 'c': /* CopyDone */ /* COPY IN correctly terminated by frontend */ fe_eof = true; ! return; case 'f': /* CopyFail */ ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), --- 458,464 ---- case 'c': /* CopyDone */ /* COPY IN correctly terminated by frontend */ fe_eof = true; ! return bytesread; case 'f': /* CopyFail */ ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), *************** *** 459,598 **** avail = datasize; pq_copymsgbytes(copy_msgbuf, databuf, avail); databuf = (void *) ((char *) databuf + avail); datasize -= avail; } break; } - } - - static int - CopyGetChar(void) - { - int ch; - - switch (copy_dest) - { - case COPY_FILE: - ch = getc(copy_file); - break; - case COPY_OLD_FE: - ch = pq_getbyte(); - if (ch == EOF) - { - /* Only a \. terminator is legal EOF in old protocol */ - ereport(ERROR, - (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("unexpected EOF on client connection"))); - } - break; - case COPY_NEW_FE: - { - unsigned char cc; - - CopyGetData(&cc, 1); - if (fe_eof) - ch = EOF; - else - ch = cc; - break; - } - default: - ch = EOF; - break; - } - if (ch == EOF) - fe_eof = true; - return ch; - } - - /* - * CopyPeekChar reads a byte in "peekable" mode. - * - * after each call to CopyPeekChar, a call to CopyDonePeek _must_ - * follow, unless EOF was returned. - * - * CopyDonePeek will either take the peeked char off the stream - * (if pickup is true) or leave it on the stream (if pickup is false). - */ - static int - CopyPeekChar(void) - { - int ch; - - switch (copy_dest) - { - case COPY_FILE: - ch = getc(copy_file); - break; - case COPY_OLD_FE: - ch = pq_peekbyte(); - if (ch == EOF) - { - /* Only a \. terminator is legal EOF in old protocol */ - ereport(ERROR, - (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("unexpected EOF on client connection"))); - } - break; - case COPY_NEW_FE: - { - unsigned char cc; - - CopyGetData(&cc, 1); - if (fe_eof) - ch = EOF; - else - ch = cc; - break; - } - default: - ch = EOF; - break; - } - if (ch == EOF) - fe_eof = true; - return ch; - } - - static void - CopyDonePeek(int c, bool pickup) - { - if (fe_eof) - return; /* can't unget an EOF */ - switch (copy_dest) - { - case COPY_FILE: - if (!pickup) - { - /* We don't want to pick it up - so put it back in there */ - ungetc(c, copy_file); - } - /* If we wanted to pick it up, it's already done */ - break; - case COPY_OLD_FE: - if (pickup) - { - /* We want to pick it up */ - (void) pq_getbyte(); - } ! /* ! * If we didn't want to pick it up, just leave it where it ! * sits ! */ ! break; ! case COPY_NEW_FE: ! if (!pickup) ! { ! /* We don't want to pick it up - so put it back in there */ ! copy_msgbuf->cursor--; ! } ! /* If we wanted to pick it up, it's already done */ ! break; ! } } - /* * These functions do apply some data conversion */ --- 488,503 ---- avail = datasize; pq_copymsgbytes(copy_msgbuf, databuf, avail); databuf = (void *) ((char *) databuf + avail); + bytesread += avail; /* update the count of bytes that were + * read so far */ datasize -= avail; } break; } ! return bytesread; } /* * These functions do apply some data conversion */ *************** *** 968,980 **** } /* Set up variables to avoid per-attribute overhead. */ ! initStringInfo(&attribute_buf); initStringInfo(&line_buf); line_buf_converted = false; client_encoding = pg_get_client_encoding(); server_encoding = GetDatabaseEncoding(); copy_dest = COPY_FILE; /* default */ copy_file = NULL; copy_msgbuf = NULL; --- 873,905 ---- } /* Set up variables to avoid per-attribute overhead. */ ! initStringInfo(&attr_buf); initStringInfo(&line_buf); line_buf_converted = false; client_encoding = pg_get_client_encoding(); server_encoding = GetDatabaseEncoding(); + /* + * check if the client encoding is one of the 5 encodings + * that are not supported as a server encodings. + */ + switch (client_encoding) + { + case PG_SJIS: + case PG_BIG5: + case PG_GBK: + case PG_UHC: + case PG_GB18030: + client_encoding_only = true; + break; + default: + client_encoding_only = false; + } + + if(!csv_mode) + escape = "\\"; /* default for text format */ + copy_dest = COPY_FILE; /* default */ copy_file = NULL; copy_msgbuf = NULL; *************** *** 1105,1113 **** errmsg("could not write to file \"%s\": %m", filename))); } ! pfree(attribute_buf.data); pfree(line_buf.data); /* * Close the relation. If reading, we can release the AccessShareLock * we got; if writing, we should hold the lock until end of --- 1030,1039 ---- errmsg("could not write to file \"%s\": %m", filename))); } ! pfree(attr_buf.data); pfree(line_buf.data); + /* * Close the relation. If reading, we can release the AccessShareLock * we got; if writing, we should hold the lock until end of *************** *** 1345,1351 **** { bytea *outputbytes; ! outputbytes = DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1], value)); /* We assume the result will not have been toasted */ CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ); --- 1271,1278 ---- { bytea *outputbytes; ! outputbytes = ! DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1], value)); /* We assume the result will not have been toasted */ CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ); *************** *** 1395,1404 **** if (copy_attname) { /* error is relevant to a particular column */ ! limit_printout_length(&attribute_buf); errcontext("COPY %s, line %d, column %s: \"%s\"", copy_relname, copy_lineno, copy_attname, ! attribute_buf.data); } else { --- 1322,1331 ---- if (copy_attname) { /* error is relevant to a particular column */ ! limit_printout_length(&attr_buf); errcontext("COPY %s, line %d, column %s: \"%s\"", copy_relname, copy_lineno, copy_attname, ! attr_buf.data); } else { *************** *** 1406,1411 **** --- 1333,1340 ---- if (line_buf_converted || client_encoding == server_encoding) { + /* Strip off the newline */ + *(line_buf.data + line_buf.len - 1) = '\0'; limit_printout_length(&line_buf); errcontext("COPY %s, line %d: \"%s\"", copy_relname, copy_lineno, *************** *** 1485,1491 **** Oid in_func_oid; Datum *values; char *nulls; - bool done = false; bool isnull; ResultRelInfo *resultRelInfo; EState *estate = CreateExecutorState(); /* for ExecConstraints() */ --- 1414,1419 ---- *************** *** 1496,1501 **** --- 1424,1434 ---- ExprContext *econtext; /* used for ExecEvalExpr for default atts */ MemoryContext oldcontext = CurrentMemoryContext; ErrorContextCallback errcontext; + int *attr_offsets; + int null_print_len; /* length of null print */ + bool no_more_data; + ListCell *cur; + tupDesc = RelationGetDescr(rel); attr = tupDesc->attrs; *************** *** 1607,1613 **** } } ! /* Prepare to catch AFTER triggers. */ AfterTriggerBeginQuery(); /* --- 1540,1548 ---- } } ! /* ! * Prepare to catch AFTER triggers. ! */ AfterTriggerBeginQuery(); /* *************** *** 1671,1676 **** --- 1606,1612 ---- values = (Datum *) palloc(num_phys_attrs * sizeof(Datum)); nulls = (char *) palloc(num_phys_attrs * sizeof(char)); + attr_offsets = (int *) palloc(num_phys_attrs * sizeof(int)); /* Make room for a PARAM_EXEC value for domain constraint checks */ if (hasConstraints) *************** *** 1691,1712 **** errcontext.previous = error_context_stack; error_context_stack = &errcontext; ! /* on input just throw the header line away */ ! if (header_line) { ! copy_lineno++; ! done = CopyReadLine(quote, escape) ; } ! while (!done) { bool skip_tuple; Oid loaded_oid = InvalidOid; CHECK_FOR_INTERRUPTS(); - copy_lineno++; - /* Reset the per-tuple exprcontext */ ResetPerTupleExprContext(estate); --- 1627,1681 ---- errcontext.previous = error_context_stack; error_context_stack = &errcontext; ! /* ! * initialize buffered scan variables. ! */ ! if(csv_mode) ! { ! in_quote = false; ! last_was_esc = false; ! } ! ! null_print_len = strlen(null_print); ! ! /* Set up data buffer to hold a chunk of data */ ! MemSet(input_buf, ' ', COPY_BUF_SIZE * sizeof(char)); ! input_buf[COPY_BUF_SIZE] = '\0'; ! ! no_more_data = false; /* no more input data to read from file or FE */ ! line_done = true; ! buf_done = false; ! ! do { ! size_t bytesread = 0; ! ! /* read a chunk of data into the buffer */ ! if (!binary) ! { ! bytesread = CopyGetData(input_buf, COPY_BUF_SIZE); ! buf_done = false; ! ! /* set buffer pointers to beginning of the buffer */ ! begloc = input_buf; ! buffer_index = 0; } ! /* ! * continue if some bytes were read or if we didn't reach EOF. if we ! * both reached EOF _and_ no bytes were read, quit the loop we are ! * done ! */ ! if (bytesread > 0 || !fe_eof || binary) ! { ! ! while (!buf_done) { bool skip_tuple; Oid loaded_oid = InvalidOid; CHECK_FOR_INTERRUPTS(); /* Reset the per-tuple exprcontext */ ResetPerTupleExprContext(estate); *************** *** 1716,1746 **** /* Initialize all values for row to NULL */ MemSet(values, 0, num_phys_attrs * sizeof(Datum)); MemSet(nulls, 'n', num_phys_attrs * sizeof(char)); if (!binary) { - CopyReadResult result = NORMAL_ATTR; - char *string; - ListCell *cur; /* Actually read the line into memory here */ ! done = csv_mode ? ! CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL); /* ! * EOF at start of line means we're done. If we see EOF after ! * some characters, we act as though it was newline followed ! * by EOF, ie, process the line and then exit loop on next ! * iteration. */ ! if (done && line_buf.len == 0) break; if (file_has_oids) { /* can't be in CSV mode here */ ! string = CopyReadAttribute(delim, null_print, ! &result, &isnull); if (isnull) ereport(ERROR, --- 1685,1729 ---- /* Initialize all values for row to NULL */ MemSet(values, 0, num_phys_attrs * sizeof(Datum)); MemSet(nulls, 'n', num_phys_attrs * sizeof(char)); + /* reset attribute pointers */ + MemSet(attr_offsets, 0, num_phys_attrs * sizeof(int)); if (!binary) { /* Actually read the line into memory here */ ! line_done = csv_mode ? ! CopyReadLineCSV(bytesread, quote, escape) : ! CopyReadLineText(bytesread, escape); /* ! * if finished processing data line - increment line count. ! * Otherwise, if eof is not yet reached, we skip att parsing ! * and read more data. But if eof _was_ reached it means ! * that the original last data line is defective and ! * we want to catch that error later on. */ ! if (line_done) ! copy_lineno++; ! else if (!fe_eof || end_marker ) break; + if (header_line) + { + line_buf.len = 0; /* we can reset line buffer now. */ + line_buf.data[0] = '\0'; + line_buf.cursor = 0; + header_line = false; + continue; + } + if (file_has_oids) { + char *oid_string; + /* can't be in CSV mode here */ ! oid_string = CopyReadOidAttr(delim, null_print, null_print_len, ! &isnull); if (isnull) ereport(ERROR, *************** *** 1750,1756 **** { copy_attname = "oid"; loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin, ! CStringGetDatum(string))); if (loaded_oid == InvalidOid) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), --- 1733,1739 ---- { copy_attname = "oid"; loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin, ! CStringGetDatum(oid_string))); if (loaded_oid == InvalidOid) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), *************** *** 1759,1792 **** } } ! /* Loop to read the user attributes on the line. */ foreach(cur, attnumlist) { int attnum = lfirst_int(cur); int m = attnum - 1; ! /* ! * If prior attr on this line was ended by newline, ! * complain. ! */ ! if (result != NORMAL_ATTR) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("missing data for column \"%s\"", ! NameStr(attr[m]->attname)))); ! if (csv_mode) ! { ! string = CopyReadAttributeCSV(delim, null_print, quote, ! escape, &result, &isnull); ! if (result == UNTERMINATED_FIELD) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unterminated CSV quoted field"))); ! } else ! string = CopyReadAttribute(delim, null_print, ! &result, &isnull); if (csv_mode && isnull && force_notnull[m]) { --- 1742,1770 ---- } } ! /* parse all the attribute in the data line */ ! if(csv_mode) ! CopyReadAttributesCSV(delim, null_print, quote, escape, null_print_len, ! nulls, attnumlist, attr_offsets, num_phys_attrs, attr); ! else ! CopyReadAttributesText(delim, escape, null_print, null_print_len, ! nulls, attnumlist, attr_offsets, num_phys_attrs, attr); ! ! /* ! * Loop to read the user attributes on the line. ! */ foreach(cur, attnumlist) { int attnum = lfirst_int(cur); int m = attnum - 1; + char *string; ! string = attr_buf.data + attr_offsets[m]; ! if (nulls[m] == ' ') ! isnull = false; else ! isnull = true; if (csv_mode && isnull && force_notnull[m]) { *************** *** 1806,1823 **** copy_attname = NULL; } } - - /* - * Complain if there are more fields on the input line. - * - * Special case: if we're reading a zero-column table, we won't - * yet have called CopyReadAttribute() at all; so no error if - * line is empty. - */ - if (result == NORMAL_ATTR && line_buf.len != 0) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("extra data after last expected column"))); } else { --- 1784,1789 ---- *************** *** 1828,1834 **** fld_count = CopyGetInt16(); if (CopyGetEof() || fld_count == -1) { ! done = true; break; } --- 1794,1801 ---- fld_count = CopyGetInt16(); if (CopyGetEof() || fld_count == -1) { ! buf_done = true; ! no_more_data = true; break; } *************** *** 1885,1891 **** nulls[defmap[i]] = ' '; } ! /* Next apply any domain constraints */ if (hasConstraints) { ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0]; --- 1852,1860 ---- nulls[defmap[i]] = ' '; } ! /* ! * Next apply any domain constraints ! */ if (hasConstraints) { ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0]; *************** *** 1912,1924 **** } } ! /* And now we can form the input tuple. */ tuple = heap_formtuple(tupDesc, values, nulls); if (oids && file_has_oids) HeapTupleSetOid(tuple, loaded_oid); ! /* Triggers and stuff need to be invoked in query context. */ MemoryContextSwitchTo(oldcontext); skip_tuple = false; --- 1881,1897 ---- } } ! /* ! * And now we can form the input tuple. ! */ tuple = heap_formtuple(tupDesc, values, nulls); if (oids && file_has_oids) HeapTupleSetOid(tuple, loaded_oid); ! /* ! * Triggers and stuff need to be invoked in query context. ! */ MemoryContextSwitchTo(oldcontext); skip_tuple = false; *************** *** 1945,1955 **** /* Place tuple in tuple slot */ ExecStoreTuple(tuple, slot, InvalidBuffer, false); ! /* Check the constraints of the tuple */ if (rel->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); ! /* OK, store the tuple and create index entries for it */ simple_heap_insert(rel, tuple); if (resultRelInfo->ri_NumIndices > 0) --- 1918,1932 ---- /* Place tuple in tuple slot */ ExecStoreTuple(tuple, slot, InvalidBuffer, false); ! /* ! * Check the constraints of the tuple ! */ if (rel->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); ! /* ! * OK, store the tuple and create index entries for it ! */ simple_heap_insert(rel, tuple); if (resultRelInfo->ri_NumIndices > 0) *************** *** 1958,1976 **** /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, tuple); } } ! /* Done, clean up */ error_context_stack = errcontext.previous; MemoryContextSwitchTo(oldcontext); ! /* Execute AFTER STATEMENT insertion triggers */ ExecASInsertTriggers(estate, resultRelInfo); ! /* Handle queued AFTER triggers */ AfterTriggerEndQuery(estate); pfree(values); pfree(nulls); --- 1935,1972 ---- /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, tuple); } + + line_buf.len = 0; /* we can reset line buffer now. */ + line_buf.data[0] = '\0'; + line_buf.cursor = 0; + } /* end while(!buf_done) */ + } /* end if (bytesread > 0 || !fe_eof) */ + else + /* no bytes read, end of data */ + { + no_more_data = true; } + } while (!no_more_data); ! /* ! * Done, clean up ! */ error_context_stack = errcontext.previous; MemoryContextSwitchTo(oldcontext); ! /* ! * Execute AFTER STATEMENT insertion triggers ! */ ExecASInsertTriggers(estate, resultRelInfo); ! /* ! * Handle queued AFTER triggers ! */ AfterTriggerEndQuery(estate); + pfree(attr_offsets); + pfree(values); pfree(nulls); *************** *** 1990,2262 **** /* ! * Read the next input line and stash it in line_buf, with conversion to ! * server encoding. ! * ! * Result is true if read was terminated by EOF, false if terminated ! * by newline. */ static bool ! CopyReadLine(char * quote, char * escape) { ! bool result; ! bool change_encoding = (client_encoding != server_encoding); ! int c; ! int mblen; ! int j; ! unsigned char s[2]; char *cvt; ! bool in_quote = false, last_was_esc = false, csv_mode = false; ! char quotec = '\0', escapec = '\0'; ! ! if (quote) ! { ! csv_mode = true; ! quotec = quote[0]; ! escapec = escape[0]; ! /* ignore special escape processing if it's the same as quotec */ ! if (quotec == escapec) ! escapec = '\0'; ! } ! ! ! s[1] = 0; ! ! /* reset line_buf to empty */ ! line_buf.len = 0; ! line_buf.data[0] = '\0'; ! line_buf.cursor = 0; /* mark that encoding conversion hasn't occurred yet */ line_buf_converted = false; ! /* set default status */ ! result = false; /* ! * In this loop we only care for detecting newlines (\r and/or \n) and ! * the end-of-copy marker (\.). ! * ! * In Text mode, for backwards compatibility we allow ! * backslashes to escape newline characters. Backslashes other than ! * the end marker get put into the line_buf, since CopyReadAttribute ! * does its own escape processing. ! * ! * In CSV mode, CR and NL inside q quoted field are just part of the ! * data value and are put in line_buf. We keep just enough state ! * to know if we are currently in a quoted field or not. ! * ! * These four characters, and only these four, are assumed the same in ! * frontend and backend encodings. ! * ! * We do not assume that second and later bytes of a frontend ! * multibyte character couldn't look like ASCII characters. */ ! for (;;) { ! c = CopyGetChar(); ! if (c == EOF) { ! result = true; ! break; } ! if (csv_mode) { /* ! * Dealing with quotes and escapes here is mildly tricky. If the ! * quote char is also the escape char, there's no problem - we ! * just use the char as a toggle. If they are different, we need ! * to ensure that we only take account of an escape inside a quoted ! * field and immediately preceding a quote char, and not the ! * second in a escape-escape sequence. */ ! if (in_quote && c == escapec) ! last_was_esc = ! last_was_esc; ! if (c == quotec && ! last_was_esc) ! in_quote = ! in_quote; ! if (c != escapec) ! last_was_esc = false; /* ! * updating the line count for embedded CR and/or LF chars is ! * necessarily a little fragile - this test is probably about ! * the best we can do. */ ! if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n')) ! copy_lineno++; ! } ! if (!in_quote && c == '\r') { ! if (eol_type == EOL_NL) { ! if (! csv_mode) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("literal carriage return found in data"), ! errhint("Use \"\\r\" to represent carriage return."))); ! else ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unquoted carriage return found in CSV data"), ! errhint("Use quoted CSV field to represent carriage return."))); } ! /* Check for \r\n on first line, _and_ handle \r\n. */ ! if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL) { ! int c2 = CopyPeekChar(); ! if (c2 == '\n') ! { ! CopyDonePeek(c2, true); /* eat newline */ ! eol_type = EOL_CRNL; } else { ! /* found \r, but no \n */ if (eol_type == EOL_CRNL) { ! if (!csv_mode) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("literal carriage return found in data"), ! errhint("Use \"\\r\" to represent carriage return."))); else ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unquoted carriage return found in data"), ! errhint("Use quoted CSV field to represent carriage return."))); ! } /* ! * if we got here, it is the first line and we didn't ! * get \n, so put it back */ ! CopyDonePeek(c2, false); ! eol_type = EOL_CR; } } - break; } ! if (!in_quote && c == '\n') { ! if (eol_type == EOL_CR || eol_type == EOL_CRNL) ! { ! if (!csv_mode) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("literal newline found in data"), ! errhint("Use \"\\n\" to represent newline."))); ! else ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unquoted newline found in data"), ! errhint("Use quoted CSV field to represent newline."))); ! } - eol_type = EOL_NL; - break; - } ! if ((line_buf.len == 0 || !csv_mode) && c == '\\') ! { ! int c2; ! ! if (csv_mode) ! c2 = CopyPeekChar(); ! else ! c2 = c = CopyGetChar(); ! if (c2 == EOF) { ! result = true; ! if (csv_mode) ! CopyDonePeek(c2, true); break; } ! if (c2 == '.') ! { ! if (csv_mode) ! CopyDonePeek(c2, true); /* allow keep calling GetChar() */ ! if (eol_type == EOL_CRNL) ! { ! c = CopyGetChar(); ! if (c == '\n') ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("end-of-copy marker does not match previous newline style"))); ! if (c != '\r') ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("end-of-copy marker corrupt"))); } - c = CopyGetChar(); - if (c != '\r' && c != '\n') - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("end-of-copy marker corrupt"))); - if ((eol_type == EOL_NL && c != '\n') || - (eol_type == EOL_CRNL && c != '\n') || - (eol_type == EOL_CR && c != '\r')) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("end-of-copy marker does not match previous newline style"))); /* ! * In protocol version 3, we should ignore anything after ! * \. up to the protocol end of copy data. (XXX maybe ! * better not to treat \. as special?) */ ! if (copy_dest == COPY_NEW_FE) { ! while (c != EOF) ! c = CopyGetChar(); } ! result = true; /* report EOF */ break; } ! if (csv_mode) ! CopyDonePeek(c2, false); /* not a dot, so put it back */ else ! /* not EOF mark, so emit \ and following char literally */ ! appendStringInfoCharMacro(&line_buf, '\\'); } ! appendStringInfoCharMacro(&line_buf, c); /* ! * When client encoding != server, must be careful to read the ! * extra bytes of a multibyte character exactly, since the ! * encoding might not ensure they don't look like ASCII. When the ! * encodings are the same, we need not do this, since no server ! * encoding we use has ASCII-like following bytes. */ ! if (change_encoding) ! { ! s[0] = c; ! mblen = pg_encoding_mblen(client_encoding, s); ! for (j = 1; j < mblen; j++) ! { ! c = CopyGetChar(); ! if (c == EOF) { ! result = true; break; } - appendStringInfoCharMacro(&line_buf, c); } ! if (result) ! break; /* out of outer loop */ } - } /* end of outer loop */ ! /* Done reading the line. Convert it to server encoding. */ ! if (change_encoding) { cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data, line_buf.len); --- 1986,2424 ---- /* ! * Finds the next TEXT line that is in the input buffer and loads ! * it into line_buf. Returns an indication if the line that was read ! * is complete (if an unescaped line-end was encountered). If we ! * reached the end of buffer before the whole line was written into the ! * line buffer then returns false. */ static bool ! CopyReadLineText(size_t bytesread, char *escape) { ! int linesize; ! bool transcode = (client_encoding != server_encoding); char *cvt; ! char escapec = '\0'; /* mark that encoding conversion hasn't occurred yet */ line_buf_converted = false; ! /* ! * set the escape char for text format ('\\' by default). ! */ ! escapec = escape[0]; /* ! * Detect end of line type if not already detected. */ ! if (eol_type == EOL_UNKNOWN) { ! bool eol_detected = DetectLineEnd(bytesread, NULL, escape); ! ! if (!eol_detected) { ! /* load entire input buffer into line buf, and quit */ ! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE); ! buf_done = true; ! return false; ! } } ! /* ! * Special case: eol is CRNL, last byte of previous buffer was an ! * unescaped CR and 1st byte of current buffer is NL. We check for ! * that here. ! */ ! if (eol_type == EOL_CRNL) ! { ! /* if we started scanning from the 1st byte of the buffer */ ! if (begloc == input_buf) ! { ! /* and had a CR in last byte of prev buf */ ! if (cr_in_prevbuf) ! { ! /* ! * if this 1st byte in buffer is 2nd byte of line end sequence ! * (linefeed) ! */ ! if (*begloc == eol_ch[1]) { /* ! * load that one linefeed byte and indicate we are done ! * with the data line */ + appendBinaryStringInfo(&line_buf, begloc, 1); + buffer_index++; + begloc++; + esc_in_prevbuf = false; + cr_in_prevbuf = false; + return true; + } + } ! cr_in_prevbuf = false; ! } ! } /* ! * (we need a loop so that if eol_ch is found, but prev ch is backslash, ! * we can search for the next eol_ch) */ ! while (true) ! { ! /* reached end of buffer */ ! if ((endloc = scanTextLine(begloc, eol_ch[0], bytesread - buffer_index)) == NULL) ! { ! linesize = COPY_BUF_SIZE - (begloc - input_buf); ! appendBinaryStringInfo(&line_buf, begloc, linesize); ! if (line_buf.len > 1) { ! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */ ! ! if (*last_ch == escapec) { ! esc_in_prevbuf = true; ! ! if (line_buf.len > 2) ! { ! last_ch--; ! if (*last_ch == escapec) ! esc_in_prevbuf = false; } ! } ! else if (*last_ch == '\r') { ! if (eol_type == EOL_CRNL) ! cr_in_prevbuf = true; ! } ! } ! line_done = false; ! buf_done = true; ! break; } else + /* found the 1st eol ch in input_buf. */ { ! bool eol_found = true; ! bool eol_escaped = true; ! ! /* ! * Load that piece of data (potentially a data line) into the line buffer, ! * and update the pointers for the next scan. ! */ ! linesize = endloc - begloc + 1; ! appendBinaryStringInfo(&line_buf, begloc, linesize); ! buffer_index += linesize; ! begloc = endloc + 1; ! if (eol_type == EOL_CRNL) { ! /* check if there is a '\n' after the '\r' */ ! if (*(endloc + 1) == '\n') ! { ! /* this is a line end */ ! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */ ! buffer_index++; ! begloc++; ! } else ! /* just a CR, not a line end */ ! eol_found = false; } /* ! * in some cases, this end of line char happens to be the ! * last character in the buffer. we need to catch that. */ ! if (buffer_index >= bytesread) ! buf_done = true; ! ! /* ! * Check if the 1st end of line ch is escaped. ! */ ! if (endloc != input_buf) /* can we look 1 char back? */ ! { ! if (*(endloc - 1) != escapec) /* prev char is not an escape */ ! eol_escaped = false; ! else ! /* prev char is an escape */ ! { ! if (endloc != (input_buf + 1)) /* can we look another ! * char back? */ ! { ! /* it's a double escape char, so it's not an escape */ ! if (*(endloc - 2) == escapec) ! eol_escaped = false; ! /* else it's a single escape char, so EOL is ascaped */ ! } ! else ! { ! /* we need to check in the last buffer */ ! if (esc_in_prevbuf) ! eol_escaped = false; } } } ! else ! /* ! * this eol ch is first ch in buffer, check for escape in prev buf ! */ { ! if (!esc_in_prevbuf) ! eol_escaped = false; } ! esc_in_prevbuf = false; /* reset variable */ ! /* ! * if eol was found, and it isn't escaped, line is done ! */ ! if ((eol_escaped == false) && eol_found) { ! line_done = true; break; } ! else ! /* stay in the loop and process some more data. */ ! line_done = false; ! ! } /* end of found eol_ch */ ! } ! /* ! * Done reading the line. Convert it to server encoding. ! */ ! if (transcode) ! { ! cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data, ! line_buf.len); ! if (cvt != line_buf.data) ! { ! /* transfer converted data back to line_buf */ ! line_buf.len = 0; ! line_buf.data[0] = '\0'; ! line_buf.cursor = 0; ! appendBinaryStringInfo(&line_buf, cvt, strlen(cvt)); ! } ! } ! ! /* indicate that conversion had occured */ ! line_buf_converted = true; ! ! /* ! * check if this line is an end marker -- "\." ! */ ! end_marker = false; ! ! switch (eol_type) ! { ! case EOL_NL: ! if (!strcmp(line_buf.data, "\\.\n")) ! end_marker = true; ! break; ! case EOL_CR: ! if (!strcmp(line_buf.data, "\\.\r")) ! end_marker = true; ! break; ! case EOL_CRNL: ! if (!strcmp(line_buf.data, "\\.\r\n")) ! end_marker = true; ! break; ! case EOL_UNKNOWN: ! break; ! } ! ! if (end_marker) ! { ! fe_eof = true; ! /* we don't want to process a \. as data line, want to quit. */ ! line_done = false; ! buf_done = true; ! } ! ! return line_done; ! } ! ! /* ! * Finds the next CSV line that is in the input buffer and loads ! * it into line_buf. Returns an indication if the line that was read ! * is complete (if an unescaped line-end was encountered). If we ! * reached the end of buffer before the whole line was written into the ! * line buffer then returns false. ! */ ! static bool ! CopyReadLineCSV(size_t bytesread, char *quote, char *escape) ! { ! int linesize; ! char *cvt; ! char quotec = '\0', ! escapec = '\0'; ! bool transcode = (client_encoding != server_encoding); ! ! /* mark that encoding conversion hasn't occurred yet */ ! line_buf_converted = false; ! ! escapec = escape[0]; ! quotec = quote[0]; ! ! /* ignore special escape processing if it's the same as quotec */ ! if (quotec == escapec) ! escapec = '\0'; ! ! /* ! * Detect end of line type if not already detected. ! */ ! if (eol_type == EOL_UNKNOWN) ! { ! bool eol_detected = DetectLineEnd(bytesread, quote, escape); ! ! if (!eol_detected) ! { ! /* load entire input buffer into line buf, and quit */ ! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE); ! line_done = false; ! buf_done = true; ! ! return line_done; ! } ! } ! ! /* ! * Special case: eol is CRNL, last byte of previous buffer was an ! * unescaped CR and 1st byte of current buffer is NL. We check for ! * that here. ! */ ! if (eol_type == EOL_CRNL) ! { ! /* if we started scanning from the 1st byte of the buffer */ ! if (begloc == input_buf) ! { ! /* and had a CR in last byte of prev buf */ ! if (cr_in_prevbuf) ! { ! /* ! * if this 1st byte in buffer is 2nd byte of line end sequence ! * (linefeed) ! */ ! if (*begloc == eol_ch[1]) ! { ! /* ! * load that one linefeed byte and indicate we are done ! * with the data line ! */ ! appendBinaryStringInfo(&line_buf, begloc, 1); ! buffer_index++; ! begloc++; ! ! line_done = true; ! esc_in_prevbuf = false; ! cr_in_prevbuf = false; ! ! return line_done; ! } ! } ! ! cr_in_prevbuf = false; ! } } /* ! * (we need a loop so that if eol_ch is found, but we are in quotes, ! * we can search for the next eol_ch) */ ! while (true) ! { ! /* reached end of buffer */ ! if ((endloc = scanCSVLine(begloc, eol_ch[0], escapec, quotec, bytesread - buffer_index)) == NULL) ! { ! linesize = COPY_BUF_SIZE - (begloc - input_buf); ! appendBinaryStringInfo(&line_buf, begloc, linesize); ! ! if (last_was_esc) ! esc_in_prevbuf = true; ! ! if (line_buf.len > 1) ! { ! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */ ! ! if (*last_ch == '\r') { ! if (eol_type == EOL_CRNL) ! cr_in_prevbuf = true; } ! } ! ! line_done = false; ! buf_done = true; break; } + else + /* found 1st eol char in input_buf. */ + { + bool eol_found = true; ! /* ! * Load that piece of data (potentially a data line) into the line buffer, ! * and update the pointers for the next scan. ! */ ! linesize = endloc - begloc + 1; ! appendBinaryStringInfo(&line_buf, begloc, linesize); ! buffer_index += linesize; ! begloc = endloc + 1; ! ! /* end of line only if not in quotes */ ! if(in_quote) ! { ! line_done = false; ! /* buf done, but still in quote */ ! if (buffer_index >= bytesread) ! { ! buf_done = true; ! break; ! } ! } ! else ! { ! /* if dos eol, check for '\n' after the '\r' */ ! if (eol_type == EOL_CRNL) ! { ! if (*(endloc + 1) == '\n') ! { ! /* this is a line end */ ! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */ ! buffer_index++; ! begloc++; ! } else ! /* just a CR, not a line end */ ! eol_found = false; } ! /* ! * in some cases, this end of line char happens to be the ! * last character in the buffer. we need to catch that. ! */ ! if (buffer_index >= bytesread) ! buf_done = true; /* ! * if eol was found line is done */ ! if (eol_found) { ! line_done = true; break; } } ! } /* end of found eol_ch */ } ! /* ! * Done reading the line. Convert it to server encoding. ! */ ! if (transcode) { cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data, line_buf.len); *************** *** 2265,2285 **** /* transfer converted data back to line_buf */ line_buf.len = 0; line_buf.data[0] = '\0'; appendBinaryStringInfo(&line_buf, cvt, strlen(cvt)); } } ! /* Now it's safe to use the buffer in error messages */ line_buf_converted = true; ! return result; } /* * Return decimal value for a hexadecimal digit */ static ! int GetDecimalFromHex(char hex) { if (isdigit(hex)) return hex - '0'; --- 2427,2480 ---- /* transfer converted data back to line_buf */ line_buf.len = 0; line_buf.data[0] = '\0'; + line_buf.cursor = 0; appendBinaryStringInfo(&line_buf, cvt, strlen(cvt)); } } ! /* indicate that conversion had occured */ line_buf_converted = true; ! /* ! * check if this line is an end marker -- "\." ! */ ! end_marker = false; ! ! switch (eol_type) ! { ! case EOL_NL: ! if (!strcmp(line_buf.data, "\\.\n")) ! end_marker = true; ! break; ! case EOL_CR: ! if (!strcmp(line_buf.data, "\\.\r")) ! end_marker = true; ! break; ! case EOL_CRNL: ! if (!strcmp(line_buf.data, "\\.\r\n")) ! end_marker = true; ! break; ! case EOL_UNKNOWN: ! break; ! } ! ! if (end_marker) ! { ! fe_eof = true; ! /* we don't want to process a \. as data line, want to quit. */ ! line_done = false; ! buf_done = true; ! } ! ! return line_done; } /* * Return decimal value for a hexadecimal digit */ static ! int ! GetDecimalFromHex(char hex) { if (isdigit(hex)) return hex - '0'; *************** *** 2287,2344 **** return tolower(hex) - 'a' + 10; } ! /*---------- ! * Read the value of a single attribute, performing de-escaping as needed. ! * ! * delim is the column delimiter string (must be just one byte for now). ! * null_print is the null marker string. Note that this is compared to ! * the pre-de-escaped input string. ! * ! * *result is set to indicate what terminated the read: ! * NORMAL_ATTR: column delimiter ! * END_OF_LINE: end of line ! * In either case, the string read up to the terminator is returned. ! * ! * *isnull is set true or false depending on whether the input matched ! * the null marker. Note that the caller cannot check this since the ! * returned string will be the post-de-escaping equivalent, which may ! * look the same as some valid data string. ! *---------- */ ! static char * ! CopyReadAttribute(const char *delim, const char *null_print, ! CopyReadResult *result, bool *isnull) { char c; ! char delimc = delim[0]; ! int start_cursor = line_buf.cursor; ! int end_cursor; ! int input_len; ! /* reset attribute_buf to empty */ ! attribute_buf.len = 0; ! attribute_buf.data[0] = '\0'; ! /* set default status */ ! *result = END_OF_LINE; ! for (;;) { ! end_cursor = line_buf.cursor; ! if (line_buf.cursor >= line_buf.len) ! break; ! c = line_buf.data[line_buf.cursor++]; ! if (c == delimc) { ! *result = NORMAL_ATTR; ! break; } ! if (c == '\\') { ! if (line_buf.cursor >= line_buf.len) ! break; ! c = line_buf.data[line_buf.cursor++]; ! switch (c) { case '0': case '1': --- 2482,2771 ---- return tolower(hex) - 'a' + 10; } ! /* ! * Detected the eol type by looking at the first data row. ! * Possible eol types are NL, CR, or CRNL. If eol type was ! * detected, it is set and a boolean true is returned to ! * indicated detection was successful. If the first data row ! * is longer than the input buffer, we return false and will ! * try again in the next buffer. */ ! static bool ! DetectLineEnd(size_t bytesread, char *quote, char *escape) { + int index = 0; char c; ! char quotec = '\0', ! escapec = '\0'; ! bool csv = false; ! if (quote) /* CSV format */ ! { ! csv = true; ! quotec = quote[0]; ! escapec = escape[0]; ! /* ignore special escape processing if it's the same as quotec */ ! if (quotec == escapec) ! escapec = '\0'; ! } ! while (index < COPY_BUF_SIZE) ! { ! c = input_buf[index]; ! if(csv) { ! if (in_quote && c == escapec) ! last_was_esc = !last_was_esc; ! if (c == quotec && !last_was_esc) ! in_quote = !in_quote; ! if (c != escapec) ! last_was_esc = false; ! } ! ! if (c == '\n') { ! if(!csv || (csv && !in_quote)) ! { ! eol_type = EOL_NL; ! eol_ch[0] = '\n'; ! eol_ch[1] = '\0'; ! ! in_quote = false; ! last_was_esc = false; ! return true; ! } } ! if (c == '\r') { ! if(!csv || (csv && !in_quote)) ! { ! if (input_buf[index + 1] == '\n') /* always safe */ ! { ! eol_type = EOL_CRNL; ! eol_ch[0] = '\r'; ! eol_ch[1] = '\n'; ! } ! else ! { ! eol_type = EOL_CR; ! eol_ch[0] = '\r'; ! eol_ch[1] = '\0'; ! } ! ! in_quote = false; ! last_was_esc = false; ! return true; ! } ! } ! ! index++; ! } ! ! return false; ! } ! ! /* ! * Read all TEXT attributes. Attributes are parsed from line_buf and ! * inserted (all at once) to attr_buf, while saving pointers to ! * each attribute's starting position. ! * ! * When this routine finishes execution both the nulls array and ! * the attr_offsets array are updated. The attr_offsets will include ! * the offset from the beginning of the attribute array of which ! * each attribute begins. If a specific attribute is not used for this ! * COPY command (ommitted from the column list), a value of 0 will be assigned. ! * For example: for table foo(a,b,c,d,e) and COPY foo(a,b,e) ! * attr_offsets may look something like this after this routine ! * returns: [0,20,0,0,55]. That means that column "a" value starts ! * at byte offset 0, "b" in 20 and "e" in 55, in attr_buf. ! * ! * In the attribute buffer (attr_buf) each attribute ! * is terminated with a '\0', and therefore by using the attr_offsets ! * array we could point to a beginning of an attribute and have it ! * behave as a C string, much like previously done in COPY. ! * ! * Another aspect to improving performance is reducing the frequency ! * of data load into buffers. The original COPY read attribute code ! * loaded a character at a time. In here we try to load a chunk of data ! * at a time. Usually a chunk will include a full data row ! * (unless we have an escaped delim). That effectively reduces the number of ! * loads by a factor of number of bytes per row. This improves performance ! * greatly, unfortunately it add more complexity to the code. ! * ! * Global participants in parsing logic: ! * ! * line_buf.cursor -- an offset from beginning of the line buffer ! * that indicates where we are about to begin the next scan. Note that ! * if we have WITH OIDS this cursor is already shifted past the first ! * OID attribute. ! * ! * attr_buf.cursor -- an offset from the beginning of the ! * attribute buffer that indicates where the current attribute begins. ! */ ! static void ! CopyReadAttributesText(const char *delim, const char *escape, const char *null_print, ! int null_print_len, char *nulls, List *attnumlist, ! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr) ! { ! char delimc = delim[0]; /* delimiter character */ ! char escapec = escape[0]; /* escape character */ ! char *scan_start; /* pointer to line buffer for scan start. */ ! char *scan_end; /* pointer to line buffer where char was found */ ! int attr_pre_len; /* attr raw len, before processing escapes */ ! int attr_post_len; /* current attr len after escaping */ ! int m; /* attribute index being parsed */ ! int bytes_remaining;/* num bytes remaining to be scanned in line buf */ ! int chunk_start; /* offset to beginning of line chunk to load */ ! int chunk_len; /* length of chunk of data to load to attr buf */ ! int oct_val; /* byte value for octal escapes */ ! int hex_val; /* byte value for hexadecimal escapes */ ! char hexchar; /* char that appears after \x for hex values */ ! int attnum; /* attribute number being parsed */ ! ListCell *cur; /* cursor to attribute list used for this COPY */ ! int attribute; ! ! /* ! * init variables for attribute scan ! */ ! attr_buf.len = 0; ! attr_buf.data[0] = '\0'; ! attr_buf.cursor = 0; ! /* cursor is now > 0 if we copy WITH OIDS */ ! scan_start = line_buf.data + line_buf.cursor; ! cur = list_head(attnumlist); ! attnum = lfirst_int(cur); ! m = attnum - 1; ! chunk_start = line_buf.cursor; ! chunk_len = 0; ! attr_pre_len = 0; ! attr_post_len = 0; ! ! /* ! * Scan through the line buffer to read all attributes data ! */ ! while (line_buf.cursor < line_buf.len) ! { ! bytes_remaining = line_buf.len - line_buf.cursor; ! ! if ((scan_end = scanTextAttr(scan_start, delimc, escapec, bytes_remaining)) ! == NULL) ! { ! /* GOT TO END OF LINE BUFFER */ ! ! if (cur == NULL) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("extra data after last expected column"))); ! ! attnum = lfirst_int(cur); ! m = attnum - 1; ! ! /* don't count eol char(s) in attr and chunk len calculation */ ! if (eol_type == EOL_CRNL) ! { ! attr_pre_len += bytes_remaining - 2; ! chunk_len = line_buf.len - chunk_start - 2; ! } ! else ! { ! attr_pre_len += bytes_remaining - 1; ! chunk_len = line_buf.len - chunk_start - 1; ! } ! ! /* check if this is a NULL value or data value (assumed NULL) */ ! if (attr_pre_len == null_print_len ! && ! strncmp(line_buf.data + line_buf.len - attr_pre_len - 1, null_print, attr_pre_len) ! == 0) ! nulls[m] = 'n'; ! else ! nulls[m] = ' '; ! ! attr_offsets[m] = attr_buf.cursor; ! ! ! /* load the last chunk, the whole buffer in most cases */ ! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len); ! ! line_buf.cursor += attr_pre_len + 2; /* skip eol char and ! * '\0' to exit loop */ ! ! if (lnext(cur) != NULL) ! { ! /* ! * For an empty data line, the previous COPY code will ! * fail it during the conversion stage. We can fail it here ! * already, but then we will fail the regression tests b/c ! * of a different error message. that's why we return so we ! * can get the same error message that regress expects. ahh... ! * this conditional is unnecessary and should be removed soon. ! */ ! if (line_buf.len > 1) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("missing data for column \"%s\"", ! NameStr(attr[m + 1]->attname)))); ! else ! return; ! } ! } ! else ! /* FOUND A DELIMITER OR ESCAPE */ ! { ! if (cur == NULL) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("extra data after last expected column"))); ! ! if (*scan_end == delimc) /* found a delimiter */ ! { ! attnum = lfirst_int(cur); ! m = attnum - 1; ! ! /* (we don't include the delimiter ch in length) */ ! attr_pre_len += scan_end - scan_start; ! /* (we don't include the delimiter ch in length) */ ! attr_post_len += scan_end - scan_start; ! ! /* check if this is a null print or data (assumed NULL) */ ! if (attr_pre_len == null_print_len ! && ! strncmp(scan_end - attr_pre_len, null_print, attr_pre_len) ! == 0) ! nulls[m] = 'n'; ! else ! nulls[m] = ' '; ! ! /* set the pointer to next attribute position */ ! attr_offsets[m] = attr_buf.cursor; ! ! /* ! * update buffer cursors to our current location, +1 to skip ! * the delimc ! */ ! line_buf.cursor = scan_end - line_buf.data + 1; ! attr_buf.cursor += attr_post_len + 1; ! ! /* prepare scan for next attr */ ! scan_start = line_buf.data + line_buf.cursor; ! cur = lnext(cur); ! attr_pre_len = 0; ! attr_post_len = 0; ! } ! else ! /* found an escape character */ ! { ! char nextc = *(scan_end + 1); ! char newc; ! int skip = 2; ! ! chunk_len = (scan_end - line_buf.data) - chunk_start + 1; ! ! /* load a chunk of data */ ! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len); ! ! switch (nextc) { case '0': case '1': *************** *** 2348,2445 **** case '5': case '6': case '7': ! /* handle \013 */ ! { ! int val; ! val = OCTVALUE(c); ! if (line_buf.cursor < line_buf.len) ! { ! c = line_buf.data[line_buf.cursor]; ! if (ISOCTAL(c)) ! { ! line_buf.cursor++; ! val = (val << 3) + OCTVALUE(c); ! if (line_buf.cursor < line_buf.len) { ! c = line_buf.data[line_buf.cursor]; ! if (ISOCTAL(c)) { ! line_buf.cursor++; ! val = (val << 3) + OCTVALUE(c); } } ! } ! } ! c = val & 0377; ! } break; case 'x': /* Handle \x3F */ ! if (line_buf.cursor < line_buf.len) ! { ! char hexchar = line_buf.data[line_buf.cursor]; if (isxdigit(hexchar)) { ! int val = GetDecimalFromHex(hexchar); ! line_buf.cursor++; ! if (line_buf.cursor < line_buf.len) ! { ! hexchar = line_buf.data[line_buf.cursor]; if (isxdigit(hexchar)) { ! line_buf.cursor++; ! val = (val << 4) + GetDecimalFromHex(hexchar); ! } ! } ! c = val & 0xff; } } break; case 'b': ! c = '\b'; break; case 'f': ! c = '\f'; break; case 'n': ! c = '\n'; break; case 'r': ! c = '\r'; break; case 't': ! c = '\t'; break; case 'v': ! c = '\v'; break; /* ! * in all other cases, take the char after '\' ! * literally */ } ! } ! appendStringInfoCharMacro(&attribute_buf, c); } ! /* check whether raw input matched null marker */ ! input_len = end_cursor - start_cursor; ! if (input_len == strlen(null_print) && ! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0) *isnull = true; ! else ! *isnull = false; ! return attribute_buf.data; ! } /* ! * Read the value of a single attribute in CSV mode, * performing de-escaping as needed. Escaping does not follow the normal * PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage. * --- 2775,2948 ---- case '5': case '6': case '7': ! oct_val = OCTVALUE(nextc); ! nextc = *(scan_end + 2); ! /* ! * (no need for out bad access check since line if ! * buffered) ! */ ! if (ISOCTAL(nextc)) { ! skip++; ! oct_val = (oct_val << 3) + OCTVALUE(nextc); ! nextc = *(scan_end + 3); ! if (ISOCTAL(nextc)) { ! skip++; ! oct_val = (oct_val << 3) + OCTVALUE(nextc); } } ! newc = oct_val & 0377; /* the escaped byte value */ break; case 'x': /* Handle \x3F */ ! hexchar = *(scan_end + 2); if (isxdigit(hexchar)) { ! skip++; ! hex_val = GetDecimalFromHex(hexchar); ! hexchar = *(scan_end + 3); if (isxdigit(hexchar)) { ! skip++; ! hex_val = (hex_val << 4) + GetDecimalFromHex(hexchar); } + newc = hex_val & 0xff; } + else /* "\x" with no hex value */ + newc = nextc; break; case 'b': ! newc = '\b'; break; case 'f': ! newc = '\f'; break; case 'n': ! newc = '\n'; break; case 'r': ! newc = '\r'; break; case 't': ! newc = '\t'; break; case 'v': ! newc = '\v'; ! break; ! default: ! if (nextc == delimc) ! newc = delimc; ! else if (nextc == escapec) ! newc = escapec; ! else ! /* no escape sequence, take next char literaly */ ! newc = nextc; break; + } + + /* update to current length, add escape and escaped chars */ + attr_pre_len += scan_end - scan_start + 2; + /* update to current length, escaped char */ + attr_post_len += scan_end - scan_start + 1; /* ! * Need to get rid of the escape character. This is done by ! * loading the chunk up to including the escape character ! * into the attribute buffer. Then overwritting the backslash ! * with the escaped sequence or char, and continuing to scan ! * from *after* the char than is after the escape in line buf. */ + *(attr_buf.data + attr_buf.len - 1) = newc; + line_buf.cursor = scan_end - line_buf.data + skip; + scan_start = scan_end + skip; + chunk_start = line_buf.cursor; + chunk_len = 0; } ! ! } /* end delimiter/backslash */ ! ! } /* end line buffer scan. */ ! ! /* ! * Replace all delimiters with NULL for string termination. ! * NOTE: only delimiters (NOT necessarily all delimc) are replaced. ! * Example (delimc = '|'): ! * - Before: f 1 | f \| 2 | f 3 ! * - After : f 1 \0 f | 2 \0 f 3 ! */ ! for (attribute = 1; attribute < num_phys_attrs; attribute++) ! { ! if (attr_offsets[attribute] != 0) ! *(attr_buf.data + attr_offsets[attribute] - 1) = '\0'; } ! } ! ! ! /* ! * Read a binary attribute ! */ ! static Datum ! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, ! Oid typioparam, int32 typmod, ! bool *isnull) ! { ! int32 fld_size; ! Datum result; ! ! fld_size = CopyGetInt32(); ! if (CopyGetEof()) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unexpected EOF in COPY data"))); ! if (fld_size == -1) ! { *isnull = true; ! return (Datum) 0; ! } ! if (fld_size < 0) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("invalid field size"))); ! /* reset attr_buf to empty, and load raw data in it */ ! attr_buf.len = 0; ! attr_buf.data[0] = '\0'; ! attr_buf.cursor = 0; ! ! enlargeStringInfo(&attr_buf, fld_size); ! ! CopyGetData(attr_buf.data, fld_size); ! if (CopyGetEof()) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unexpected EOF in COPY data"))); + attr_buf.len = fld_size; + attr_buf.data[fld_size] = '\0'; + + /* Call the column type's binary input converter */ + result = FunctionCall3(flinfo, + PointerGetDatum(&attr_buf), + ObjectIdGetDatum(typioparam), + Int32GetDatum(typmod)); + + /* Trouble if it didn't eat the whole buffer */ + if (attr_buf.cursor != attr_buf.len) + ereport(ERROR, + (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), + errmsg("incorrect binary data format"))); + + *isnull = false; + return result; + } /* ! * Read all the attributes of the data line in CSV mode, * performing de-escaping as needed. Escaping does not follow the normal * PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage. * *************** *** 2448,2472 **** * * null_print is the null marker string. Note that this is compared to * the pre-de-escaped input string (thus if it is quoted it is not a NULL). - * - * *result is set to indicate what terminated the read: - * NORMAL_ATTR: column delimiter - * END_OF_LINE: end of line - * UNTERMINATED_FIELD no quote detected at end of a quoted field - * - * In any case, the string read up to the terminator (or end of file) - * is returned. - * - * *isnull is set true or false depending on whether the input matched - * the null marker. Note that the caller cannot check this since the - * returned string will be the post-de-escaping equivalent, which may - * look the same as some valid data string. *---------- */ ! ! static char * ! CopyReadAttributeCSV(const char *delim, const char *null_print, char *quote, ! char *escape, CopyReadResult *result, bool *isnull) { char delimc = delim[0]; char quotec = quote[0]; --- 2951,2962 ---- * * null_print is the null marker string. Note that this is compared to * the pre-de-escaped input string (thus if it is quoted it is not a NULL). *---------- */ ! static void ! CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote, ! char *escape, int null_print_len, char *nulls, List *attnumlist, ! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr) { char delimc = delim[0]; char quotec = quote[0]; *************** *** 2477,2502 **** int input_len; bool in_quote = false; bool saw_quote = false; ! /* reset attribute_buf to empty */ ! attribute_buf.len = 0; ! attribute_buf.data[0] = '\0'; - /* set default status */ - *result = END_OF_LINE; for (;;) { end_cursor = line_buf.cursor; if (line_buf.cursor >= line_buf.len) break; c = line_buf.data[line_buf.cursor++]; /* unquoted field delimiter */ if (!in_quote && c == delimc) { ! *result = NORMAL_ATTR; ! break; } /* start of quoted field (or part of field) */ --- 2967,3047 ---- int input_len; bool in_quote = false; bool saw_quote = false; + int attnum; /* attribute number being parsed */ + int m; /* attribute index being parsed */ + ListCell *cur; /* cursor to attribute list used for this COPY */ ! /* ! * init variables for attribute scan ! */ ! attr_buf.len = 0; ! attr_buf.data[0] = '\0'; ! attr_buf.cursor = 0; ! ! cur = list_head(attnumlist); ! attnum = lfirst_int(cur); ! m = attnum - 1; ! input_len = 0; for (;;) { end_cursor = line_buf.cursor; + + /* finished processing attributes in line */ if (line_buf.cursor >= line_buf.len) + { + /* check whether raw input matched null marker */ + input_len = end_cursor - start_cursor; + if (!saw_quote && input_len == null_print_len && + strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0) + nulls[m] = 'n'; + else + nulls[m] = ' '; + + if (in_quote) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("unterminated CSV quoted field"))); + + if (lnext(cur) != NULL) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("missing data for column \"%s\"", + NameStr(attr[m + 1]->attname)))); break; + } + c = line_buf.data[line_buf.cursor++]; /* unquoted field delimiter */ if (!in_quote && c == delimc) { ! /* check whether raw input matched null marker */ ! input_len = end_cursor - start_cursor; ! if (!saw_quote && input_len == null_print_len && ! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0) ! nulls[m] = 'n'; ! else ! nulls[m] = ' '; ! ! /* terminate attr string with '\0' */ ! appendStringInfoCharMacro(&attr_buf, '\0'); ! attr_buf.cursor++; ! ! /* setup next attribute scan */ ! cur = lnext(cur); ! ! if (cur == NULL) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("extra data after last expected column"))); ! ! attnum = lfirst_int(cur); ! m = attnum - 1; ! attr_offsets[m] = attr_buf.cursor; ! start_cursor = line_buf.cursor; ! continue; } /* start of quoted field (or part of field) */ *************** *** 2520,2527 **** if (nextc == escapec || nextc == quotec) { ! appendStringInfoCharMacro(&attribute_buf, nextc); line_buf.cursor++; continue; } } --- 3065,3073 ---- if (nextc == escapec || nextc == quotec) { ! appendStringInfoCharMacro(&attr_buf, nextc); line_buf.cursor++; + attr_buf.cursor++; continue; } } *************** *** 2537,2616 **** in_quote = false; continue; } ! appendStringInfoCharMacro(&attribute_buf, c); } - if (in_quote) - *result = UNTERMINATED_FIELD; - - /* check whether raw input matched null marker */ - input_len = end_cursor - start_cursor; - if (!saw_quote && input_len == strlen(null_print) && - strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0) - *isnull = true; - else - *isnull = false; - - return attribute_buf.data; } /* ! * Read a binary attribute */ ! static Datum ! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, ! Oid typioparam, int32 typmod, bool *isnull) { ! int32 fld_size; ! Datum result; ! ! fld_size = CopyGetInt32(); ! if (CopyGetEof()) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("unexpected EOF in COPY data"))); ! if (fld_size == -1) ! { ! *isnull = true; ! return (Datum) 0; } ! if (fld_size < 0) ! ereport(ERROR, ! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), ! errmsg("invalid field size"))); ! /* reset attribute_buf to empty, and load raw data in it */ ! attribute_buf.len = 0; ! attribute_buf.data[0] = '\0'; ! attribute_buf.cursor = 0; ! enlargeStringInfo(&attribute_buf, fld_size); - CopyGetData(attribute_buf.data, fld_size); - if (CopyGetEof()) - ereport(ERROR, - (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - errmsg("unexpected EOF in COPY data"))); ! attribute_buf.len = fld_size; ! attribute_buf.data[fld_size] = '\0'; ! /* Call the column type's binary input converter */ ! result = FunctionCall3(flinfo, ! PointerGetDatum(&attribute_buf), ! ObjectIdGetDatum(typioparam), ! Int32GetDatum(typmod)); - /* Trouble if it didn't eat the whole buffer */ - if (attribute_buf.cursor != attribute_buf.len) - ereport(ERROR, - (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), - errmsg("incorrect binary data format"))); - *isnull = false; - return result; - } /* * Send text representation of one attribute, with conversion and escaping --- 3083,3149 ---- in_quote = false; continue; } ! appendStringInfoCharMacro(&attr_buf, c); ! attr_buf.cursor++; } } /* ! * Read the first attribute. This is mainly used to maintain support ! * for an OID column. All the rest of the columns will be read at once with ! * CopyReadAttributesText. */ ! static char * ! CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len, bool *isnull) { ! char delimc = delim[0]; ! char *start_loc = line_buf.data + line_buf.cursor; ! char *end_loc; ! int attr_len = 0; ! int bytes_remaining; ! ! /* reset attribute buf to empty */ ! attr_buf.len = 0; ! attr_buf.data[0] = '\0'; ! attr_buf.cursor = 0; ! ! /* # of bytes that were not yet processed in this line */ ! bytes_remaining = line_buf.len - line_buf.cursor; ! ! /* got to end of line */ ! if ((end_loc = scanTextLine(start_loc, delimc, bytes_remaining)) == NULL) ! { ! attr_len = bytes_remaining - 1; /* don't count '\n' in len calculation */ ! appendBinaryStringInfo(&attr_buf, start_loc, attr_len); ! line_buf.cursor += attr_len + 2; /* skip '\n' and '\0' */ } ! else ! /* found a delimiter */ ! { ! /* ! * (we don't care if delim was preceded with a backslash, because it's ! * an invalid OID anyway) ! */ ! attr_len = end_loc - start_loc; /* we don't include the delimiter ch */ ! appendBinaryStringInfo(&attr_buf, start_loc, attr_len); ! line_buf.cursor += attr_len + 1; ! } ! /* check whether raw input matched null marker */ ! if (attr_len == null_print_len && strncmp(start_loc, null_print, attr_len) == 0) ! *isnull = true; ! else ! *isnull = false; ! return attr_buf.data; ! } /* * Send text representation of one attribute, with conversion and escaping *************** *** 2800,2802 **** --- 3333,3436 ---- return attnums; } + + /* + * The following are custom versions of the string function strchr(). + * As opposed to the original strchr which searches through + * a string until the target character is found, or a NULL is + * found, this version will not return when a NULL is found. + * Instead it will search through a pre-defined length of + * bytes and will return only if the target character(s) is reached. + * + * If our client encoding is not a supported server encoding, we + * know that it is not safe to look at each character as trailing + * byte in a multibyte character may be a 7-bit ASCII equivalent. + * Therefore we use pg_encoding_mblen to skip to the end of the + * character. + * + * returns: + * pointer to c - if c is located within the string. + * NULL - if c was not found in specified length of search. Note: + * this DOESN'T mean that a '\0' was reached. + */ + char * + scanTextLine(const char *s, char eol, size_t len) + { + const char *start; + + if (client_encoding_only && !line_buf_converted) + { + int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s); + + for (start = s; *s != eol && s < start + len; s += mblen) + mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s); + + return ((*s == eol) ? (char *) s : NULL); + } + else + return memchr(s,eol,len); + } + + + char * + scanCSVLine(const char *s, char eol, char escapec, char quotec, size_t len) + { + const char *start; + + if (client_encoding_only && !line_buf_converted) + { + int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s); + + for (start = s; *s != eol && s < start + len; s += mblen) + { + if (in_quote && *s == escapec) + last_was_esc = !last_was_esc; + if (*s == quotec && !last_was_esc) + in_quote = !in_quote; + if (*s != escapec) + last_was_esc = false; + + mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s); + } + } + else + /* safe to scroll byte by byte */ + { + for (start = s; *s != eol && s < start + len; s++) + { + if (in_quote && *s == escapec) + last_was_esc = !last_was_esc; + if (*s == quotec && !last_was_esc) + in_quote = !in_quote; + if (*s != escapec) + last_was_esc = false; + } + + } + + if(*s != escapec) /* found eol_ch */ + last_was_esc = false; + + return ((*s == eol) ? (char *) s : NULL); + } + + /* + * Scan for 1 of 2 characters necessary for attribute parsing. No need + * for checking for multi-byte characters since conversion already + * happened. + */ + char * + scanTextAttr(const char *s, char c1, char c2, size_t len) + { + const char *start; + + for (start = s; *s != c1 && *s != c2 && s < start + len; s++) + ; + + return (*s != '\0' ? (char *) s : NULL); + } + + + + +