xml-content-2006-2.patch

diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml
index 94b46a6..752eadf 100644
--- a/doc/src/sgml/datatype.sgml
+++ b/doc/src/sgml/datatype.sgml
@@ -4208,9 +4208,11 @@ a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11
    <para>
     The <type>xml</type> type can store well-formed
     <quote>documents</quote>, as defined by the XML standard, as well
-    as <quote>content</quote> fragments, which are defined by the
-    production <literal>XMLDecl? content</literal> in the XML
-    standard.  Roughly, this means that content fragments can have
+    as <quote>content</quote> fragments, which are defined by reference to the
+    more permissive
+    <ulink url="https://www.w3.org/TR/2010/REC-xpath-datamodel-20101214/#DocumentNode"><quote>document node</quote></ulink>
+    of the XQuery and XPath data model.
+    Roughly, this means that content fragments can have
     more than one top-level element or character node.  The expression
     <literal><replaceable>xmlvalue</replaceable> IS DOCUMENT</literal>
     can be used to evaluate whether a particular <type>xml</type>
@@ -4282,24 +4284,9 @@ SET XML OPTION { DOCUMENT | CONTENT };
 SET xmloption TO { DOCUMENT | CONTENT };
 </synopsis>
     The default is <literal>CONTENT</literal>, so all forms of XML
-    data are allowed except as noted below.
+    data are allowed.
    </para>
 
-   <note>
-    <para>
-     In the SQL:2006 and later standard, the <literal>CONTENT</literal> form
-     is a proper superset of the <literal>DOCUMENT</literal> form, and so the
-     default XML option setting would allow casting to XML from character
-     strings in either form. <productname>PostgreSQL</productname>, however,
-     uses the SQL:2003 definition in which <literal>CONTENT</literal> form
-     cannot contain a document type declaration. Therefore, there is no one
-     setting of the XML option that will allow casting to XML from every valid
-     character string. The default will work almost always, but for a document
-     with a DTD, it will be necessary to change the XML option or
-     use <literal>XMLPARSE</literal> specifying <literal>DOCUMENT</literal>.
-    </para>
-   </note>
-
    </sect2>
 
    <sect2>
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index e947613..32908c1 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -569,7 +569,7 @@ X056	Advanced table mapping: data mapping			YES
 X057	Advanced table mapping: metadata mapping			YES	
 X058	Advanced table mapping: base64 encoding of binary strings			YES	
 X059	Advanced table mapping: hex encoding of binary strings			YES	
-X060	XMLParse: character string input and CONTENT option			YES	uses SQL:2003 definition, so a document with a DTD cannot be parsed as CONTENT
+X060	XMLParse: character string input and CONTENT option			YES	
 X061	XMLParse: character string input and DOCUMENT option			YES	
 X065	XMLParse: BLOB input and CONTENT option			NO	
 X066	XMLParse: BLOB input and DOCUMENT option			NO	
diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c
index 28b3eaa..23f7dcb 100644
--- a/src/backend/utils/adt/xml.c
+++ b/src/backend/utils/adt/xml.c
@@ -110,6 +110,8 @@ struct PgXmlErrorContext
 	/* current error status and accumulated message, if any */
 	bool		err_occurred;
 	StringInfoData err_buf;
+	/* set in xml_parse if trying to parse CONTENT; reset in err hdlr if DTD */
+	bool		tentatively_content;
 	/* previous libxml error handling state (saved by pg_xml_init) */
 	xmlStructuredErrorFunc saved_errfunc;
 	void	   *saved_errcxt;
@@ -120,6 +122,7 @@ struct PgXmlErrorContext
 static xmlParserInputPtr xmlPgEntityLoader(const char *URL, const char *ID,
 				  xmlParserCtxtPtr ctxt);
 static void xml_errorHandler(void *data, xmlErrorPtr error);
+static bool xml_onlyPrologNodes(xmlDocPtr doc);
 static void xml_ereport_by_code(int level, int sqlcode,
 					const char *msg, int errcode);
 static void chopStringInfoNewlines(StringInfo str);
@@ -1002,6 +1005,7 @@ pg_xml_init(PgXmlStrictness strictness)
 	errcxt->magic = ERRCXT_MAGIC;
 	errcxt->strictness = strictness;
 	errcxt->err_occurred = false;
+	errcxt->tentatively_content = false;
 	initStringInfo(&errcxt->err_buf);
 
 	/*
@@ -1433,6 +1437,9 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
 	xmlChar    *string;
 	xmlChar    *utf8string;
 	PgXmlErrorContext *xmlerrcxt;
+	int			errcode_document_parse = ERRCODE_INVALID_XML_DOCUMENT;
+	char const *errmsg_document_parse = "invalid XML document";
+	char const *errmsg_content_parse = "invalid XML content";
 	volatile xmlParserCtxtPtr ctxt = NULL;
 	volatile xmlDocPtr doc = NULL;
 
@@ -1457,6 +1464,7 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
 			xml_ereport(xmlerrcxt, ERROR, ERRCODE_OUT_OF_MEMORY,
 						"could not allocate parser context");
 
+try_other_xmloption:
 		if (xmloption_arg == XMLOPTION_DOCUMENT)
 		{
 			/*
@@ -1472,8 +1480,8 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
 								 XML_PARSE_NOENT | XML_PARSE_DTDATTR
 								 | (preserve_whitespace ? 0 : XML_PARSE_NOBLANKS));
 			if (doc == NULL || xmlerrcxt->err_occurred)
-				xml_ereport(xmlerrcxt, ERROR, ERRCODE_INVALID_XML_DOCUMENT,
-							"invalid XML document");
+				xml_ereport(xmlerrcxt, ERROR, errcode_document_parse,
+							errmsg_document_parse);
 		}
 		else
 		{
@@ -1497,11 +1505,42 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
 			/* allow empty content */
 			if (*(utf8string + count))
 			{
+				xmlerrcxt->tentatively_content = true;
 				res_code = xmlParseBalancedChunkMemory(doc, NULL, NULL, 0,
 													   utf8string + count, NULL);
+				/*
+				 * If tentatively_content is now false, the error handler has
+				 * reset it, which will only happen for one reason: a DTD was
+				 * found in the input. The SQL/XML:2003 definition of CONTENT
+				 * is not satisfied by a document with a DTD, which is a bit of
+				 * a wart, as it means the CONTENT type is not a proper superset
+				 * of DOCUMENT. SQL/XML:2006 and later fix that, by declaring
+				 * that any DOCUMENT value is indeed also a CONTENT value. That
+				 * definition is more useful, as CONTENT becomes usable for
+				 * parsing input of unknown form (think pg_restore). Without
+				 * bringing the whole implementation along to 2006+, we can
+				 * provide the 2006+ definition of CONTENT easily enough, by
+				 * catching this error case and simply retrying the parse
+				 * as DOCUMENT.
+				 */
+				if ( ! xmlerrcxt->tentatively_content )
+				{
+					/*
+					 * xmlParseBalanced... was not passed our ctxt; it
+					 * creates one internally for its own use, so we have
+					 * no need to clear/reset ctxt here.
+					 */
+					xmlFreeDoc(doc);
+					xmloption_arg = XMLOPTION_DOCUMENT;
+					errcode_document_parse = ERRCODE_INVALID_XML_CONTENT;
+					errmsg_document_parse = errmsg_content_parse;
+					goto try_other_xmloption;
+				}
 				if (res_code != 0 || xmlerrcxt->err_occurred)
+				{
 					xml_ereport(xmlerrcxt, ERROR, ERRCODE_INVALID_XML_CONTENT,
-								"invalid XML content");
+								errmsg_content_parse);
+				}
 			}
 		}
 	}
@@ -1699,6 +1738,27 @@ xml_errorHandler(void *data, xmlErrorPtr error)
 	switch (domain)
 	{
 		case XML_FROM_PARSER:
+			/*
+			 * One very specific error from the parser will be intercepted here
+			 * if strictness is PG_XML_STRICTNESS_WELLFORMED (meaning the call
+			 * originates in xml_parse), the input is being tentatively parsed
+			 * as content, and the parser has stumbled on what looks like a DTD.
+			 * For that case, just change tentatively_content to false and
+			 * return, and xml_parse will retry the input as a document.
+			 * See xml_parse for motivation.
+			 */
+			if (xmlerrcxt->strictness == PG_XML_STRICTNESS_WELLFORMED
+				&& xmlerrcxt->tentatively_content
+				&& error->code == XML_ERR_NAME_REQUIRED
+				&& input != NULL
+				&& input->cur != NULL
+				&& 0 == xmlStrncmp(input->cur, BAD_CAST"!DOCTYPE", 8)
+				&& xml_onlyPrologNodes(ctxt->myDoc))
+			{
+				xmlerrcxt->tentatively_content = false;
+				return;
+			}
+			/* FALLTHROUGH */
 		case XML_FROM_NONE:
 		case XML_FROM_MEMORY:
 		case XML_FROM_IO:
@@ -1811,6 +1871,52 @@ xml_errorHandler(void *data, xmlErrorPtr error)
 	pfree(errorBuf);
 }
 
+/*
+ * Check (from the error handler) that the document node's only children already
+ * parsed are those that can precede a DTD.
+ *
+ * To deliver the SQL/XML:2006+ definition of 'content', the error handler above
+ * can cause a reparse as 'document' if a tentative parse as 'content' trips
+ * over something that looks like a DTD.  Without this check, goofy input like
+ * <a><!DOCTYPE b></a> could also cause an attempted 'document' reparse, which
+ * would correctly fail, but the error message could give the wrong reason.
+ *
+ * This explores the document that is being constructed. If anything about its
+ * structure is not as expected, the return is false, so the libxml error will
+ * be reported, with no other attempt at special handling.
+ */
+static bool
+xml_onlyPrologNodes(xmlDocPtr doc)
+{
+	xmlNodePtr n;
+
+	if ( doc == NULL )
+		return false;
+
+	n = doc->children; /* parseBalancedChunk creates a temporary "pseudoroot" */
+	if ( n == NULL  ||  n->type != XML_ELEMENT_NODE )
+		return false;
+	if ( n->name == NULL  ||  0 != xmlStrcmp(n->name, BAD_CAST"pseudoroot") )
+		return false;
+
+	for ( n = n->children; n != NULL; n = n->next )
+	{
+		switch (n->type)
+		{
+			case XML_PI_NODE:
+			case XML_COMMENT_NODE:
+				continue;
+			case XML_TEXT_NODE:
+				if ( xmlIsBlankNode(n) )
+					continue;
+				/* FALLTHROUGH */
+			default:
+				return false;
+		}
+	}
+	return true;
+}
+
 
 /*
  * Wrapper for "ereport" function for XML-related errors.  The "msg"
diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out
index 2085fa0..0aae600 100644
--- a/src/test/regress/expected/xml.out
+++ b/src/test/regress/expected/xml.out
@@ -532,6 +532,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  invalid XML document
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+<!DOCTYPE a><a/><b/>
+                ^
 SET XML OPTION CONTENT;
 EXECUTE foo ('<bar/>');
   xmlconcat   
@@ -545,6 +552,45 @@ EXECUTE foo ('good');
  <foo/>good
 (1 row)
 
+SELECT xml '<!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>';
+                                xml                                 
+--------------------------------------------------------------------
+ <!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<?xml version="1.0"?> <!-- hi--> <!DOCTYPE a><a/>';
+             xml              
+------------------------------
+  <!-- hi--> <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<!DOCTYPE a><a/>';
+       xml        
+------------------
+ <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+<!-- hi--> oops <!DOCTYPE a><a/>
+                 ^
+SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+<!-- hi--> <oops/> <!DOCTYPE a><a/>
+                    ^
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+<!DOCTYPE a><a/><b/>
+                ^
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');
diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out
index 7f86696..d1a03b5 100644
--- a/src/test/regress/expected/xml_1.out
+++ b/src/test/regress/expected/xml_1.out
@@ -429,11 +429,53 @@ EXECUTE foo ('<bar/>');
 ERROR:  prepared statement "foo" does not exist
 EXECUTE foo ('bad');
 ERROR:  prepared statement "foo" does not exist
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
 SET XML OPTION CONTENT;
 EXECUTE foo ('<bar/>');
 ERROR:  prepared statement "foo" does not exist
 EXECUTE foo ('good');
 ERROR:  prepared statement "foo" does not exist
+SELECT xml '<!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!-- in SQL:2006+ a doc is content too--> <?y z?...
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '<?xml version="1.0"?> <!-- hi--> <!DOCTYPE a><a/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<?xml version="1.0"?> <!-- hi--> <!DOCTYPE a><a/...
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '<!DOCTYPE a><a/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!DOCTYPE a><a/>';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');
diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out
index 510b09b..9756b6e 100644
--- a/src/test/regress/expected/xml_2.out
+++ b/src/test/regress/expected/xml_2.out
@@ -512,6 +512,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  invalid XML document
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+<!DOCTYPE a><a/><b/>
+                ^
 SET XML OPTION CONTENT;
 EXECUTE foo ('<bar/>');
   xmlconcat   
@@ -525,6 +532,45 @@ EXECUTE foo ('good');
  <foo/>good
 (1 row)
 
+SELECT xml '<!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>';
+                                xml                                 
+--------------------------------------------------------------------
+ <!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<?xml version="1.0"?> <!-- hi--> <!DOCTYPE a><a/>';
+             xml              
+------------------------------
+  <!-- hi--> <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<!DOCTYPE a><a/>';
+       xml        
+------------------
+ <!DOCTYPE a><a/>
+(1 row)
+
+SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+<!-- hi--> oops <!DOCTYPE a><a/>
+                 ^
+SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+<!-- hi--> <oops/> <!DOCTYPE a><a/>
+                    ^
+SELECT xml '<!DOCTYPE a><a/><b/>';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '<!DOCTYPE a><a/><b/>';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+<!DOCTYPE a><a/><b/>
+                ^
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');
diff --git a/src/test/regress/sql/xml.sql b/src/test/regress/sql/xml.sql
index 8057a46..71431d8 100644
--- a/src/test/regress/sql/xml.sql
+++ b/src/test/regress/sql/xml.sql
@@ -149,10 +149,17 @@ PREPARE foo (xml) AS SELECT xmlconcat('<foo/>', $1);
 SET XML OPTION DOCUMENT;
 EXECUTE foo ('<bar/>');
 EXECUTE foo ('bad');
+SELECT xml '<!DOCTYPE a><a/><b/>';
 
 SET XML OPTION CONTENT;
 EXECUTE foo ('<bar/>');
 EXECUTE foo ('good');
+SELECT xml '<!-- in SQL:2006+ a doc is content too--> <?y z?> <!DOCTYPE a><a/>';
+SELECT xml '<?xml version="1.0"?> <!-- hi--> <!DOCTYPE a><a/>';
+SELECT xml '<!DOCTYPE a><a/>';
+SELECT xml '<!-- hi--> oops <!DOCTYPE a><a/>';
+SELECT xml '<!-- hi--> <oops/> <!DOCTYPE a><a/>';
+SELECT xml '<!DOCTYPE a><a/><b/>';
 
 
 -- Test backwards parsing