variable-substitution-in-backticks-1.patch

diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index b51b11b..ad463e7 100644
*** a/doc/src/sgml/ref/psql-ref.sgml
--- b/doc/src/sgml/ref/psql-ref.sgml
*************** testdb=>
*** 770,786 ****
      </para>
  
      <para>
-     Within an argument, text that is enclosed in backquotes
-     (<literal>`</literal>) is taken as a command line that is passed to the
-     shell. The output of the command (with any trailing newline removed)
-     replaces the backquoted text.
-     </para>
- 
-     <para>
      If an unquoted colon (<literal>:</literal>) followed by a
      <application>psql</> variable name appears within an argument, it is
      replaced by the variable's value, as described in <xref
      linkend="APP-PSQL-interpolation" endterm="APP-PSQL-interpolation-title">.
      </para>
  
      <para>
--- 770,801 ----
      </para>
  
      <para>
      If an unquoted colon (<literal>:</literal>) followed by a
      <application>psql</> variable name appears within an argument, it is
      replaced by the variable's value, as described in <xref
      linkend="APP-PSQL-interpolation" endterm="APP-PSQL-interpolation-title">.
+     The forms <literal>:'<replaceable>variable_name</>'</literal> and
+     <literal>:"<replaceable>variable_name</>"</literal> described there
+     work as well.
+     </para>
+ 
+     <para>
+     Within an argument, text that is enclosed in backquotes
+     (<literal>`</literal>) is taken as a command line that is passed to the
+     shell.  The output of the command (with any trailing newline removed)
+     replaces the backquoted text.  Within the text enclosed in backquotes,
+     no special quoting or other processing occurs, except that appearances
+     of <literal>:<replaceable>variable_name</></literal> where
+     <replaceable>variable_name</> is a <application>psql</> variable name
+     are replaced by the variable's value.  Also, appearances of
+     <literal>:'<replaceable>variable_name</>'</literal> are replaced by the
+     variable's value suitably quoted to become a single shell command
+     argument.  (The latter form is almost always preferable, unless you are
+     very sure of what is in the variable.)  Because carriage return and line
+     feed characters cannot be safely quoted on all platforms, the
+     <literal>:'<replaceable>variable_name</>'</literal> form prints an
+     error message and does not substitute the variable value when such
+     characters appear in the value.
      </para>
  
      <para>
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index b06ae97..a2f1259 100644
*** a/src/bin/psql/common.c
--- b/src/bin/psql/common.c
*************** setQFout(const char *fname)
*** 116,134 ****
   * If the specified variable exists, return its value as a string (malloc'd
   * and expected to be freed by the caller); else return NULL.
   *
!  * If "escape" is true, return the value suitably quoted and escaped,
!  * as an identifier or string literal depending on "as_ident".
!  * (Failure in escaping should lead to returning NULL.)
   *
   * "passthrough" is the pointer previously given to psql_scan_set_passthrough.
   * In psql, passthrough points to a ConditionalStack, which we check to
   * determine whether variable expansion is allowed.
   */
  char *
! psql_get_variable(const char *varname, bool escape, bool as_ident,
  				  void *passthrough)
  {
! 	char	   *result;
  	const char *value;
  
  	/* In an inactive \if branch, suppress all variable substitutions */
--- 116,134 ----
   * If the specified variable exists, return its value as a string (malloc'd
   * and expected to be freed by the caller); else return NULL.
   *
!  * If "quote" isn't PQUOTE_PLAIN, then return the value suitably quoted and
!  * escaped for the specified quoting requirement.  (Failure in escaping
!  * should lead to printing an error and returning NULL.)
   *
   * "passthrough" is the pointer previously given to psql_scan_set_passthrough.
   * In psql, passthrough points to a ConditionalStack, which we check to
   * determine whether variable expansion is allowed.
   */
  char *
! psql_get_variable(const char *varname, PsqlScanQuoteType quote,
  				  void *passthrough)
  {
! 	char	   *result = NULL;
  	const char *value;
  
  	/* In an inactive \if branch, suppress all variable substitutions */
*************** psql_get_variable(const char *varname, b
*** 139,178 ****
  	if (!value)
  		return NULL;
  
! 	if (escape)
  	{
! 		char	   *escaped_value;
  
! 		if (!pset.db)
! 		{
! 			psql_error("cannot escape without active connection\n");
! 			return NULL;
! 		}
  
! 		if (as_ident)
! 			escaped_value =
! 				PQescapeIdentifier(pset.db, value, strlen(value));
! 		else
! 			escaped_value =
! 				PQescapeLiteral(pset.db, value, strlen(value));
  
! 		if (escaped_value == NULL)
! 		{
! 			const char *error = PQerrorMessage(pset.db);
  
! 			psql_error("%s", error);
! 			return NULL;
! 		}
  
! 		/*
! 		 * Rather than complicate the lexer's API with a notion of which
! 		 * free() routine to use, just pay the price of an extra strdup().
! 		 */
! 		result = pg_strdup(escaped_value);
! 		PQfreemem(escaped_value);
  	}
- 	else
- 		result = pg_strdup(value);
  
  	return result;
  }
--- 139,212 ----
  	if (!value)
  		return NULL;
  
! 	switch (quote)
  	{
! 		case PQUOTE_PLAIN:
! 			result = pg_strdup(value);
! 			break;
! 		case PQUOTE_SQL_LITERAL:
! 		case PQUOTE_SQL_IDENT:
! 			{
! 				/*
! 				 * For these cases, we use libpq's quoting functions, which
! 				 * assume the string is in the connection's client encoding.
! 				 */
! 				char	   *escaped_value;
  
! 				if (!pset.db)
! 				{
! 					psql_error("cannot escape without active connection\n");
! 					return NULL;
! 				}
  
! 				if (quote == PQUOTE_SQL_LITERAL)
! 					escaped_value =
! 						PQescapeLiteral(pset.db, value, strlen(value));
! 				else
! 					escaped_value =
! 						PQescapeIdentifier(pset.db, value, strlen(value));
  
! 				if (escaped_value == NULL)
! 				{
! 					const char *error = PQerrorMessage(pset.db);
  
! 					psql_error("%s", error);
! 					return NULL;
! 				}
  
! 				/*
! 				 * Rather than complicate the lexer's API with a notion of
! 				 * which free() routine to use, just pay the price of an extra
! 				 * strdup().
! 				 */
! 				result = pg_strdup(escaped_value);
! 				PQfreemem(escaped_value);
! 				break;
! 			}
! 		case PQUOTE_SHELL_ARG:
! 			{
! 				/*
! 				 * For this we use appendShellStringNoError, which is
! 				 * encoding-agnostic, which is fine since the shell probably
! 				 * is too.  In any case, the only special character is "'",
! 				 * which is not known to appear in valid multibyte characters.
! 				 */
! 				PQExpBufferData buf;
! 
! 				initPQExpBuffer(&buf);
! 				if (!appendShellStringNoError(&buf, value))
! 				{
! 					psql_error("shell command argument contains a newline or carriage return: \"%s\"\n",
! 							   value);
! 					free(buf.data);
! 					return NULL;
! 				}
! 				result = buf.data;
! 				break;
! 			}
! 
! 			/* No default: we want a compiler warning for missing cases */
  	}
  
  	return result;
  }
diff --git a/src/bin/psql/common.h b/src/bin/psql/common.h
index 3d8b8da..1ceb8ae 100644
*** a/src/bin/psql/common.h
--- b/src/bin/psql/common.h
***************
*** 12,22 ****
  
  #include "libpq-fe.h"
  #include "fe_utils/print.h"
  
  extern bool openQueryOutputFile(const char *fname, FILE **fout, bool *is_pipe);
  extern bool setQFout(const char *fname);
  
! extern char *psql_get_variable(const char *varname, bool escape, bool as_ident,
  				  void *passthrough);
  
  extern void psql_error(const char *fmt,...) pg_attribute_printf(1, 2);
--- 12,23 ----
  
  #include "libpq-fe.h"
  #include "fe_utils/print.h"
+ #include "fe_utils/psqlscan.h"
  
  extern bool openQueryOutputFile(const char *fname, FILE **fout, bool *is_pipe);
  extern bool setQFout(const char *fname);
  
! extern char *psql_get_variable(const char *varname, PsqlScanQuoteType quote,
  				  void *passthrough);
  
  extern void psql_error(const char *fmt,...) pg_attribute_printf(1, 2);
diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l
index 319afdc..db7a1b9 100644
*** a/src/bin/psql/psqlscanslash.l
--- b/src/bin/psql/psqlscanslash.l
*************** other			.
*** 242,249 ****
  															 yytext + 1,
  															 yyleng - 1);
  						value = cur_state->callbacks->get_variable(varname,
! 																   false,
! 																   false,
  																   cur_state->cb_passthrough);
  						free(varname);
  
--- 242,248 ----
  															 yytext + 1,
  															 yyleng - 1);
  						value = cur_state->callbacks->get_variable(varname,
! 																   PQUOTE_PLAIN,
  																   cur_state->cb_passthrough);
  						free(varname);
  
*************** other			.
*** 268,281 ****
  				}
  
  :'{variable_char}+'	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng, false);
  					*option_quote = ':';
  					unquoted_option_chars = 0;
  				}
  
  
  :\"{variable_char}+\"	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng, true);
  					*option_quote = ':';
  					unquoted_option_chars = 0;
  				}
--- 267,282 ----
  				}
  
  :'{variable_char}+'	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng,
! 											 PQUOTE_SQL_LITERAL);
  					*option_quote = ':';
  					unquoted_option_chars = 0;
  				}
  
  
  :\"{variable_char}+\"	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng,
! 											 PQUOTE_SQL_IDENT);
  					*option_quote = ':';
  					unquoted_option_chars = 0;
  				}
*************** other			.
*** 337,345 ****
  
  <xslashbackquote>{
  	/*
! 	 * backticked text: copy everything until next backquote, then evaluate.
! 	 *
! 	 * XXX Possible future behavioral change: substitute for :VARIABLE?
  	 */
  
  "`"				{
--- 338,345 ----
  
  <xslashbackquote>{
  	/*
! 	 * backticked text: copy everything until next backquote (expanding
! 	 * variable references, but doing nought else), then evaluate.
  	 */
  
  "`"				{
*************** other			.
*** 350,355 ****
--- 350,393 ----
  					BEGIN(xslasharg);
  				}
  
+ :{variable_char}+	{
+ 					/* Possible psql variable substitution */
+ 					if (cur_state->callbacks->get_variable == NULL)
+ 						ECHO;
+ 					else
+ 					{
+ 						char	   *varname;
+ 						char	   *value;
+ 
+ 						varname = psqlscan_extract_substring(cur_state,
+ 															 yytext + 1,
+ 															 yyleng - 1);
+ 						value = cur_state->callbacks->get_variable(varname,
+ 																   PQUOTE_PLAIN,
+ 																   cur_state->cb_passthrough);
+ 						free(varname);
+ 
+ 						if (value)
+ 						{
+ 							appendPQExpBufferStr(output_buf, value);
+ 							free(value);
+ 						}
+ 						else
+ 							ECHO;
+ 					}
+ 				}
+ 
+ :'{variable_char}+'	{
+ 					psqlscan_escape_variable(cur_state, yytext, yyleng,
+ 											 PQUOTE_SHELL_ARG);
+ 				}
+ 
+ :'{variable_char}*	{
+ 					/* Throw back everything but the colon */
+ 					yyless(1);
+ 					ECHO;
+ 				}
+ 
  {other}|\n		{ ECHO; }
  
  }
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index 19b3e57..27689d7 100644
*** a/src/fe_utils/psqlscan.l
--- b/src/fe_utils/psqlscan.l
*************** other			.
*** 699,706 ****
  														 yyleng - 1);
  					if (cur_state->callbacks->get_variable)
  						value = cur_state->callbacks->get_variable(varname,
! 																   false,
! 																   false,
  																   cur_state->cb_passthrough);
  					else
  						value = NULL;
--- 699,705 ----
  														 yyleng - 1);
  					if (cur_state->callbacks->get_variable)
  						value = cur_state->callbacks->get_variable(varname,
! 																   PQUOTE_PLAIN,
  																   cur_state->cb_passthrough);
  					else
  						value = NULL;
*************** other			.
*** 737,747 ****
  				}
  
  :'{variable_char}+'	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng, false);
  				}
  
  :\"{variable_char}+\"	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng, true);
  				}
  
  	/*
--- 736,748 ----
  				}
  
  :'{variable_char}+'	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng,
! 											 PQUOTE_SQL_LITERAL);
  				}
  
  :\"{variable_char}+\"	{
! 					psqlscan_escape_variable(cur_state, yytext, yyleng,
! 											 PQUOTE_SQL_IDENT);
  				}
  
  	/*
*************** psqlscan_extract_substring(PsqlScanState
*** 1415,1421 ****
   */
  void
  psqlscan_escape_variable(PsqlScanState state, const char *txt, int len,
! 						 bool as_ident)
  {
  	char	   *varname;
  	char	   *value;
--- 1416,1422 ----
   */
  void
  psqlscan_escape_variable(PsqlScanState state, const char *txt, int len,
! 						 PsqlScanQuoteType quote)
  {
  	char	   *varname;
  	char	   *value;
*************** psqlscan_escape_variable(PsqlScanState s
*** 1423,1429 ****
  	/* Variable lookup. */
  	varname = psqlscan_extract_substring(state, txt + 2, len - 3);
  	if (state->callbacks->get_variable)
! 		value = state->callbacks->get_variable(varname, true, as_ident,
  											   state->cb_passthrough);
  	else
  		value = NULL;
--- 1424,1430 ----
  	/* Variable lookup. */
  	varname = psqlscan_extract_substring(state, txt + 2, len - 3);
  	if (state->callbacks->get_variable)
! 		value = state->callbacks->get_variable(varname, quote,
  											   state->cb_passthrough);
  	else
  		value = NULL;
diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c
index d1a9ddc..dc84d32 100644
*** a/src/fe_utils/string_utils.c
--- b/src/fe_utils/string_utils.c
*************** appendByteaLiteral(PQExpBuffer buf, cons
*** 425,437 ****
--- 425,454 ----
   * arguments containing LF or CR characters.  A future major release should
   * reject those characters in CREATE ROLE and CREATE DATABASE, because use
   * there eventually leads to errors here.
+  *
+  * appendShellString() simply prints an error and dies if LF or CR appears.
+  * appendShellStringNoError() omits those characters from the result, and
+  * returns false if there were any.
   */
  void
  appendShellString(PQExpBuffer buf, const char *str)
  {
+ 	if (!appendShellStringNoError(buf, str))
+ 	{
+ 		fprintf(stderr,
+ 				_("shell command argument contains a newline or carriage return: \"%s\"\n"),
+ 				str);
+ 		exit(EXIT_FAILURE);
+ 	}
+ }
+ 
+ bool
+ appendShellStringNoError(PQExpBuffer buf, const char *str)
+ {
  #ifdef WIN32
  	int			backslash_run_length = 0;
  #endif
+ 	bool		ok = true;
  	const char *p;
  
  	/*
*************** appendShellString(PQExpBuffer buf, const
*** 442,448 ****
  		strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_./:") == strlen(str))
  	{
  		appendPQExpBufferStr(buf, str);
! 		return;
  	}
  
  #ifndef WIN32
--- 459,465 ----
  		strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_./:") == strlen(str))
  	{
  		appendPQExpBufferStr(buf, str);
! 		return ok;
  	}
  
  #ifndef WIN32
*************** appendShellString(PQExpBuffer buf, const
*** 451,460 ****
  	{
  		if (*p == '\n' || *p == '\r')
  		{
! 			fprintf(stderr,
! 					_("shell command argument contains a newline or carriage return: \"%s\"\n"),
! 					str);
! 			exit(EXIT_FAILURE);
  		}
  
  		if (*p == '\'')
--- 468,475 ----
  	{
  		if (*p == '\n' || *p == '\r')
  		{
! 			ok = false;
! 			continue;
  		}
  
  		if (*p == '\'')
*************** appendShellString(PQExpBuffer buf, const
*** 481,490 ****
  	{
  		if (*p == '\n' || *p == '\r')
  		{
! 			fprintf(stderr,
! 					_("shell command argument contains a newline or carriage return: \"%s\"\n"),
! 					str);
! 			exit(EXIT_FAILURE);
  		}
  
  		/* Change N backslashes before a double quote to 2N+1 backslashes. */
--- 496,503 ----
  	{
  		if (*p == '\n' || *p == '\r')
  		{
! 			ok = false;
! 			continue;
  		}
  
  		/* Change N backslashes before a double quote to 2N+1 backslashes. */
*************** appendShellString(PQExpBuffer buf, const
*** 524,529 ****
--- 537,544 ----
  	}
  	appendPQExpBufferStr(buf, "^\"");
  #endif   /* WIN32 */
+ 
+ 	return ok;
  }
  
  
diff --git a/src/include/fe_utils/psqlscan.h b/src/include/fe_utils/psqlscan.h
index 0cc632b..e9c8143 100644
*** a/src/include/fe_utils/psqlscan.h
--- b/src/include/fe_utils/psqlscan.h
*************** typedef enum _promptStatus
*** 48,60 ****
  	PROMPT_COPY
  } promptStatus_t;
  
  /* Callback functions to be used by the lexer */
  typedef struct PsqlScanCallbacks
  {
! 	/* Fetch value of a variable, as a pfree'able string; NULL if unknown */
  	/* This pointer can be NULL if no variable substitution is wanted */
! 	char	   *(*get_variable) (const char *varname, bool escape,
! 										   bool as_ident, void *passthrough);
  	/* Print an error message someplace appropriate */
  	/* (very old gcc versions don't support attributes on function pointers) */
  #if defined(__GNUC__) && __GNUC__ < 4
--- 48,69 ----
  	PROMPT_COPY
  } promptStatus_t;
  
+ /* Quoting request types for get_variable() callback */
+ typedef enum
+ {
+ 	PQUOTE_PLAIN,				/* just return the actual value */
+ 	PQUOTE_SQL_LITERAL,			/* add quotes to make a valid SQL literal */
+ 	PQUOTE_SQL_IDENT,			/* quote if needed to make a SQL identifier */
+ 	PQUOTE_SHELL_ARG			/* quote if needed to be safe in a shell cmd */
+ } PsqlScanQuoteType;
+ 
  /* Callback functions to be used by the lexer */
  typedef struct PsqlScanCallbacks
  {
! 	/* Fetch value of a variable, as a free'able string; NULL if unknown */
  	/* This pointer can be NULL if no variable substitution is wanted */
! 	char	   *(*get_variable) (const char *varname, PsqlScanQuoteType quote,
! 											 void *passthrough);
  	/* Print an error message someplace appropriate */
  	/* (very old gcc versions don't support attributes on function pointers) */
  #if defined(__GNUC__) && __GNUC__ < 4
diff --git a/src/include/fe_utils/psqlscan_int.h b/src/include/fe_utils/psqlscan_int.h
index b4044e8..af62f5e 100644
*** a/src/include/fe_utils/psqlscan_int.h
--- b/src/include/fe_utils/psqlscan_int.h
*************** extern char *psqlscan_extract_substring(
*** 141,146 ****
  						   const char *txt, int len);
  extern void psqlscan_escape_variable(PsqlScanState state,
  						 const char *txt, int len,
! 						 bool as_ident);
  
  #endif   /* PSQLSCAN_INT_H */
--- 141,146 ----
  						   const char *txt, int len);
  extern void psqlscan_escape_variable(PsqlScanState state,
  						 const char *txt, int len,
! 						 PsqlScanQuoteType quote);
  
  #endif   /* PSQLSCAN_INT_H */
diff --git a/src/include/fe_utils/string_utils.h b/src/include/fe_utils/string_utils.h
index 6fb7f5e..c682343 100644
*** a/src/include/fe_utils/string_utils.h
--- b/src/include/fe_utils/string_utils.h
*************** extern void appendByteaLiteral(PQExpBuff
*** 42,47 ****
--- 42,48 ----
  				   bool std_strings);
  
  extern void appendShellString(PQExpBuffer buf, const char *str);
+ extern bool appendShellStringNoError(PQExpBuffer buf, const char *str);
  extern void appendConnStrVal(PQExpBuffer buf, const char *str);
  extern void appendPsqlMetaConnect(PQExpBuffer buf, const char *dbname);