copy_encoding-20110217.patch

diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml
index 38424ad..6429a4e 100644
*** a/doc/src/sgml/ref/copy.sgml
--- b/doc/src/sgml/ref/copy.sgml
*************** COPY { <replaceable class="parameter">ta
*** 40,46 ****
      QUOTE '<replaceable class="parameter">quote_character</replaceable>'
      ESCAPE '<replaceable class="parameter">escape_character</replaceable>'
      FORCE_QUOTE { ( <replaceable class="parameter">column</replaceable> [, ...] ) | * }
!     FORCE_NOT_NULL ( <replaceable class="parameter">column</replaceable> [, ...] )
  </synopsis>
   </refsynopsisdiv>
  
--- 40,47 ----
      QUOTE '<replaceable class="parameter">quote_character</replaceable>'
      ESCAPE '<replaceable class="parameter">escape_character</replaceable>'
      FORCE_QUOTE { ( <replaceable class="parameter">column</replaceable> [, ...] ) | * }
!     FORCE_NOT_NULL ( <replaceable class="parameter">column</replaceable> [, ...] ) |
!     ENCODING '<replaceable class="parameter">encoding_name</replaceable>'
  </synopsis>
   </refsynopsisdiv>
  
*************** COPY { <replaceable class="parameter">ta
*** 282,287 ****
--- 283,300 ----
      </listitem>
     </varlistentry>
  
+    <varlistentry>
+     <term><literal>ENCODING</></term>
+     <listitem>
+      <para>
+       Specifies that the file is encoded in the <replaceable
+       class="parameter">encoding_name</replaceable>.  If this option is
+       omitted, the current client encoding is used. See the Notes below
+       for more details.
+      </para>
+     </listitem>
+    </varlistentry>
+ 
    </variablelist>
   </refsect1>
  
*************** COPY <replaceable class="parameter">coun
*** 377,384 ****
     </para>
  
     <para>
!     Input data is interpreted according to the current client encoding,
!     and output data is encoded in the current client encoding, even
      if the data does not pass through the client but is read from or
      written to a file directly by the server.
     </para>
--- 390,398 ----
     </para>
  
     <para>
!     Input data is interpreted according to <literal>ENCODING</literal>
!     option or the current client encoding, and output data is encoded
!     in <literal>ENCODING</literal> or the current client encoding, even
      if the data does not pass through the client but is read from or
      written to a file directly by the server.
     </para>
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 9f7263d..04bd71d 100644
*** a/src/backend/commands/copy.c
--- b/src/backend/commands/copy.c
*************** typedef struct CopyStateData
*** 95,102 ****
  								 * dest == COPY_NEW_FE in COPY FROM */
  	bool		fe_eof;			/* true if detected end of copy data */
  	EolType		eol_type;		/* EOL type of input */
! 	int			client_encoding;	/* remote side's character encoding */
! 	bool		need_transcoding;		/* client encoding diff from server? */
  	bool		encoding_embeds_ascii;	/* ASCII can be non-first byte? */
  
  	/* parameters from the COPY command */
--- 95,102 ----
  								 * dest == COPY_NEW_FE in COPY FROM */
  	bool		fe_eof;			/* true if detected end of copy data */
  	EolType		eol_type;		/* EOL type of input */
! 	int			file_encoding;	/* file or remote side's character encoding */
! 	bool		need_transcoding;		/* file encoding diff from server? */
  	bool		encoding_embeds_ascii;	/* ASCII can be non-first byte? */
  
  	/* parameters from the COPY command */
*************** typedef struct CopyStateData
*** 110,116 ****
  	bool		header_line;	/* CSV header line? */
  	char	   *null_print;		/* NULL marker string (server encoding!) */
  	int			null_print_len; /* length of same */
! 	char	   *null_print_client;		/* same converted to client encoding */
  	char	   *delim;			/* column delimiter (must be 1 byte) */
  	char	   *quote;			/* CSV quote char (must be 1 byte) */
  	char	   *escape;			/* CSV escape char (must be 1 byte) */
--- 110,116 ----
  	bool		header_line;	/* CSV header line? */
  	char	   *null_print;		/* NULL marker string (server encoding!) */
  	int			null_print_len; /* length of same */
! 	char	   *null_print_client;		/* same converted to file encoding */
  	char	   *delim;			/* column delimiter (must be 1 byte) */
  	char	   *quote;			/* CSV quote char (must be 1 byte) */
  	char	   *escape;			/* CSV escape char (must be 1 byte) */
*************** BeginCopy(bool is_from,
*** 848,853 ****
--- 848,854 ----
  
  	/* Allocate workspace and zero all fields */
  	cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
+ 	cstate->file_encoding = -1;
  
  	/*
  	 * We allocate everything used by a cstate in a new memory context.
*************** BeginCopy(bool is_from,
*** 964,969 ****
--- 965,983 ----
  						 errmsg("argument to option \"%s\" must be a list of column names",
  								defel->defname)));
  		}
+ 		else if (strcmp(defel->defname, "encoding") == 0)
+ 		{
+ 			if (cstate->file_encoding >= 0)
+ 				ereport(ERROR,
+ 						(errcode(ERRCODE_SYNTAX_ERROR),
+ 						 errmsg("conflicting or redundant options")));
+ 			cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
+ 			if (cstate->file_encoding < 0)
+ 				ereport(ERROR,
+ 						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ 						 errmsg("argument to option \"%s\" must be a valid encoding name",
+ 								defel->defname)));
+ 		}
  		else
  			ereport(ERROR,
  					(errcode(ERRCODE_SYNTAX_ERROR),
*************** BeginCopy(bool is_from,
*** 1246,1262 ****
  		}
  	}
  
  	/*
! 	 * Set up encoding conversion info.  Even if the client and server
! 	 * encodings are the same, we must apply pg_client_to_server() to validate
  	 * data in multibyte encodings.
  	 */
- 	cstate->client_encoding = pg_get_client_encoding();
  	cstate->need_transcoding =
! 		(cstate->client_encoding != GetDatabaseEncoding() ||
  		 pg_database_encoding_max_length() > 1);
  	/* See Multibyte encoding comment above */
! 	cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->client_encoding);
  
  	cstate->copy_dest = COPY_FILE;		/* default */
  
--- 1260,1279 ----
  		}
  	}
  
+ 	/* Use client encoding when ENCODING option is not specified. */
+ 	if (cstate->file_encoding < 0)
+ 		cstate->file_encoding = pg_get_client_encoding();
+ 
  	/*
! 	 * Set up encoding conversion info.  Even if the file and server
! 	 * encodings are the same, we must apply pg_any_to_server() to validate
  	 * data in multibyte encodings.
  	 */
  	cstate->need_transcoding =
! 		(cstate->file_encoding != GetDatabaseEncoding() ||
  		 pg_database_encoding_max_length() > 1);
  	/* See Multibyte encoding comment above */
! 	cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
  
  	cstate->copy_dest = COPY_FILE;		/* default */
  
*************** CopyTo(CopyState cstate)
*** 1494,1505 ****
  	else
  	{
  		/*
! 		 * For non-binary copy, we need to convert null_print to client
  		 * encoding, because it will be sent directly with CopySendString.
  		 */
  		if (cstate->need_transcoding)
! 			cstate->null_print_client = pg_server_to_client(cstate->null_print,
! 													 cstate->null_print_len);
  
  		/* if a header has been requested send the line */
  		if (cstate->header_line)
--- 1511,1523 ----
  	else
  	{
  		/*
! 		 * For non-binary copy, we need to convert null_print to file
  		 * encoding, because it will be sent directly with CopySendString.
  		 */
  		if (cstate->need_transcoding)
! 			cstate->null_print_client = pg_server_to_any(cstate->null_print,
! 														 cstate->null_print_len,
! 														 cstate->file_encoding);
  
  		/* if a header has been requested send the line */
  		if (cstate->header_line)
*************** CopyReadLine(CopyState cstate)
*** 2576,2583 ****
  	{
  		char	   *cvt;
  
! 		cvt = pg_client_to_server(cstate->line_buf.data,
! 								  cstate->line_buf.len);
  		if (cvt != cstate->line_buf.data)
  		{
  			/* transfer converted data back to line_buf */
--- 2594,2602 ----
  	{
  		char	   *cvt;
  
! 		cvt = pg_any_to_server(cstate->line_buf.data,
! 							   cstate->line_buf.len,
! 							   cstate->file_encoding);
  		if (cvt != cstate->line_buf.data)
  		{
  			/* transfer converted data back to line_buf */
*************** CopyReadLineText(CopyState cstate)
*** 2822,2828 ****
  			/* -----
  			 * get next character
  			 * Note: we do not change c so if it isn't \., we can fall
! 			 * through and continue processing for client encoding.
  			 * -----
  			 */
  			c2 = copy_raw_buf[raw_buf_ptr];
--- 2841,2847 ----
  			/* -----
  			 * get next character
  			 * Note: we do not change c so if it isn't \., we can fall
! 			 * through and continue processing for file encoding.
  			 * -----
  			 */
  			c2 = copy_raw_buf[raw_buf_ptr];
*************** not_end_of_copy:
*** 2936,2942 ****
  
  			mblen_str[0] = c;
  			/* All our encodings only read the first byte to get the length */
! 			mblen = pg_encoding_mblen(cstate->client_encoding, mblen_str);
  			IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
  			IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
  			raw_buf_ptr += mblen - 1;
--- 2955,2961 ----
  
  			mblen_str[0] = c;
  			/* All our encodings only read the first byte to get the length */
! 			mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
  			IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
  			IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
  			raw_buf_ptr += mblen - 1;
*************** CopyAttributeOutText(CopyState cstate, c
*** 3435,3441 ****
  	char		delimc = cstate->delim[0];
  
  	if (cstate->need_transcoding)
! 		ptr = pg_server_to_client(string, strlen(string));
  	else
  		ptr = string;
  
--- 3454,3460 ----
  	char		delimc = cstate->delim[0];
  
  	if (cstate->need_transcoding)
! 		ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
  	else
  		ptr = string;
  
*************** CopyAttributeOutText(CopyState cstate, c
*** 3508,3514 ****
  				start = ptr++;	/* we include char in next run */
  			}
  			else if (IS_HIGHBIT_SET(c))
! 				ptr += pg_encoding_mblen(cstate->client_encoding, ptr);
  			else
  				ptr++;
  		}
--- 3527,3533 ----
  				start = ptr++;	/* we include char in next run */
  			}
  			else if (IS_HIGHBIT_SET(c))
! 				ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
  			else
  				ptr++;
  		}
*************** CopyAttributeOutCSV(CopyState cstate, ch
*** 3595,3601 ****
  		use_quote = true;
  
  	if (cstate->need_transcoding)
! 		ptr = pg_server_to_client(string, strlen(string));
  	else
  		ptr = string;
  
--- 3614,3620 ----
  		use_quote = true;
  
  	if (cstate->need_transcoding)
! 		ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
  	else
  		ptr = string;
  
*************** CopyAttributeOutCSV(CopyState cstate, ch
*** 3622,3628 ****
  					break;
  				}
  				if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
! 					tptr += pg_encoding_mblen(cstate->client_encoding, tptr);
  				else
  					tptr++;
  			}
--- 3641,3647 ----
  					break;
  				}
  				if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
! 					tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
  				else
  					tptr++;
  			}
*************** CopyAttributeOutCSV(CopyState cstate, ch
*** 3646,3652 ****
  				start = ptr;	/* we include char in next run */
  			}
  			if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
! 				ptr += pg_encoding_mblen(cstate->client_encoding, ptr);
  			else
  				ptr++;
  		}
--- 3665,3671 ----
  				start = ptr;	/* we include char in next run */
  			}
  			if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
! 				ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
  			else
  				ptr++;
  		}
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 3857205..320cde5 100644
*** a/src/backend/parser/gram.y
--- b/src/backend/parser/gram.y
*************** copy_opt_item:
*** 2233,2238 ****
--- 2233,2242 ----
  				{
  					$$ = makeDefElem("force_not_null", (Node *)$4);
  				}
+ 			| ENCODING Sconst
+ 				{
+ 					$$ = makeDefElem("encoding", (Node *)makeString($2));
+ 				}
  		;
  
  /* The following exist for backward compatibility with very old versions */
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 5ee74f7..b8a2728 100644
*** a/src/backend/utils/mb/mbutils.c
--- b/src/backend/utils/mb/mbutils.c
*************** pg_encoding_max_length_sql(PG_FUNCTION_A
*** 497,510 ****
  char *
  pg_client_to_server(const char *s, int len)
  {
  	Assert(DatabaseEncoding);
  	Assert(ClientEncoding);
  
  	if (len <= 0)
  		return (char *) s;
  
! 	if (ClientEncoding->encoding == DatabaseEncoding->encoding ||
! 		ClientEncoding->encoding == PG_SQL_ASCII)
  	{
  		/*
  		 * No conversion is needed, but we must still validate the data.
--- 497,521 ----
  char *
  pg_client_to_server(const char *s, int len)
  {
+ 	Assert(ClientEncoding);
+ 
+ 	return pg_any_to_server(s, len, ClientEncoding->encoding);
+ }
+ 
+ /*
+  * convert any encoding to server encoding.
+  */
+ char *
+ pg_any_to_server(const char *s, int len, int encoding)
+ {
  	Assert(DatabaseEncoding);
  	Assert(ClientEncoding);
  
  	if (len <= 0)
  		return (char *) s;
  
! 	if (encoding == DatabaseEncoding->encoding ||
! 		encoding == PG_SQL_ASCII)
  	{
  		/*
  		 * No conversion is needed, but we must still validate the data.
*************** pg_client_to_server(const char *s, int l
*** 524,531 ****
  		 * to the parser but we have no way to convert it.	We compromise by
  		 * rejecting the data if it contains any non-ASCII characters.
  		 */
! 		if (PG_VALID_BE_ENCODING(ClientEncoding->encoding))
! 			(void) pg_verify_mbstr(ClientEncoding->encoding, s, len, false);
  		else
  		{
  			int			i;
--- 535,542 ----
  		 * to the parser but we have no way to convert it.	We compromise by
  		 * rejecting the data if it contains any non-ASCII characters.
  		 */
! 		if (PG_VALID_BE_ENCODING(encoding))
! 			(void) pg_verify_mbstr(encoding, s, len, false);
  		else
  		{
  			int			i;
*************** pg_client_to_server(const char *s, int l
*** 543,549 ****
  		return (char *) s;
  	}
  
! 	return perform_default_encoding_conversion(s, len, true);
  }
  
  /*
--- 554,564 ----
  		return (char *) s;
  	}
  
! 	if (ClientEncoding->encoding == encoding)
! 		return perform_default_encoding_conversion(s, len, true);
! 	else
! 		return (char *) pg_do_encoding_conversion(
! 			(unsigned char *) s, len, encoding, DatabaseEncoding->encoding);
  }
  
  /*
*************** pg_client_to_server(const char *s, int l
*** 552,569 ****
  char *
  pg_server_to_client(const char *s, int len)
  {
  	Assert(DatabaseEncoding);
  	Assert(ClientEncoding);
  
  	if (len <= 0)
  		return (char *) s;
  
! 	if (ClientEncoding->encoding == DatabaseEncoding->encoding ||
! 		ClientEncoding->encoding == PG_SQL_ASCII ||
  		DatabaseEncoding->encoding == PG_SQL_ASCII)
  		return (char *) s;		/* assume data is valid */
  
! 	return perform_default_encoding_conversion(s, len, false);
  }
  
  /*
--- 567,599 ----
  char *
  pg_server_to_client(const char *s, int len)
  {
+ 	Assert(ClientEncoding);
+ 
+ 	return pg_any_to_server(s, len, ClientEncoding->encoding);
+ }
+ 
+ /*
+  * convert server encoding to any encoding.
+  */
+ char *
+ pg_server_to_any(const char *s, int len, int encoding)
+ {
  	Assert(DatabaseEncoding);
  	Assert(ClientEncoding);
  
  	if (len <= 0)
  		return (char *) s;
  
! 	if (encoding == DatabaseEncoding->encoding ||
! 		encoding == PG_SQL_ASCII ||
  		DatabaseEncoding->encoding == PG_SQL_ASCII)
  		return (char *) s;		/* assume data is valid */
  
! 	if (ClientEncoding->encoding == encoding)
! 		return perform_default_encoding_conversion(s, len, false);
! 	else
! 		return (char *) pg_do_encoding_conversion(
! 			(unsigned char *) s, len, DatabaseEncoding->encoding, encoding);
  }
  
  /*
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 565b53b..85a7b2f 100644
*** a/src/include/mb/pg_wchar.h
--- b/src/include/mb/pg_wchar.h
*************** extern unsigned char *pg_do_encoding_con
*** 420,425 ****
--- 420,427 ----
  
  extern char *pg_client_to_server(const char *s, int len);
  extern char *pg_server_to_client(const char *s, int len);
+ extern char *pg_any_to_server(const char *s, int len, int encoding);
+ extern char *pg_server_to_any(const char *s, int len, int encoding);
  
  extern unsigned short BIG5toCNS(unsigned short big5, unsigned char *lc);
  extern unsigned short CNStoBIG5(unsigned short cns, unsigned char lc);
diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out
index 15cbe02..8e2bc0c 100644
*** a/src/test/regress/expected/copy2.out
--- b/src/test/regress/expected/copy2.out
*************** CONTEXT:  COPY x, line 1: "2001	231	\N	\
*** 46,55 ****
  COPY x from stdin;
  ERROR:  extra data after last expected column
  CONTEXT:  COPY x, line 1: "2002	232	40	50	60	70	80"
! -- various COPY options: delimiters, oids, NULL string
  COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
  COPY x from stdin WITH DELIMITER AS ';' NULL AS '';
! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X';
  -- check results of copy in
  SELECT * FROM x;
     a   | b  |     c      |   d    |          e           
--- 46,55 ----
  COPY x from stdin;
  ERROR:  extra data after last expected column
  CONTEXT:  COPY x, line 1: "2002	232	40	50	60	70	80"
! -- various COPY options: delimiters, oids, NULL string, encoding
  COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
  COPY x from stdin WITH DELIMITER AS ';' NULL AS '';
! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X' ENCODING 'sql_ascii';
  -- check results of copy in
  SELECT * FROM x;
     a   | b  |     c      |   d    |          e           
*************** COPY y TO stdout WITH CSV QUOTE '''' DEL
*** 187,193 ****
  Jackson, Sam|\h
  It is "perfect".|	
  ''|
! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\';
  "Jackson, Sam","\\h"
  "It is \"perfect\".","	"
  "",
--- 187,193 ----
  Jackson, Sam|\h
  It is "perfect".|	
  ''|
! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\' ENCODING 'sql_ascii';
  "Jackson, Sam","\\h"
  "It is \"perfect\".","	"
  "",
diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql
index c2e8b03..6322c8f 100644
*** a/src/test/regress/sql/copy2.sql
--- b/src/test/regress/sql/copy2.sql
*************** COPY x from stdin;
*** 72,78 ****
  2002	232	40	50	60	70	80
  \.
  
! -- various COPY options: delimiters, oids, NULL string
  COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
  500000,x,45,80,90
  500001,x,\x,\\x,\\\x
--- 72,78 ----
  2002	232	40	50	60	70	80
  \.
  
! -- various COPY options: delimiters, oids, NULL string, encoding
  COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
  500000,x,45,80,90
  500001,x,\x,\\x,\\\x
*************** COPY x from stdin WITH DELIMITER AS ';' 
*** 83,89 ****
  3000;;c;;
  \.
  
! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X';
  4000:\X:C:\X:\X
  4001:1:empty::
  4002:2:null:\X:\X
--- 83,89 ----
  3000;;c;;
  \.
  
! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X' ENCODING 'sql_ascii';
  4000:\X:C:\X:\X
  4001:1:empty::
  4002:2:null:\X:\X
*************** INSERT INTO y VALUES ('', NULL);
*** 127,133 ****
  
  COPY y TO stdout WITH CSV;
  COPY y TO stdout WITH CSV QUOTE '''' DELIMITER '|';
! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\';
  COPY y TO stdout WITH CSV FORCE QUOTE *;
  
  -- Repeat above tests with new 9.0 option syntax
--- 127,133 ----
  
  COPY y TO stdout WITH CSV;
  COPY y TO stdout WITH CSV QUOTE '''' DELIMITER '|';
! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\' ENCODING 'sql_ascii';
  COPY y TO stdout WITH CSV FORCE QUOTE *;
  
  -- Repeat above tests with new 9.0 option syntax