bpo-36673: Implement comment/PI parsing support for the TreeBuilder in ElementTree.

scoder · scoder · commit ab7341d85f03 · 2019-04-20T11:18:36.000+02:00
diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
@@ -523,8 +523,9 @@ Functions
    Parses an XML section into an element tree incrementally, and reports what's
    going on to the user.  *source* is a filename or :term:`file object`
    containing XML data.  *events* is a sequence of events to report back.  The
-   supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and
-   ``"end-ns"`` (the "ns" events are used to get detailed namespace
+   supported events are the strings ``"start"``, ``"end"``, ``"comment"``,
+   ``"pi"``, ``"start-ns"`` and ``"end-ns"``
+   (the "ns" events are used to get detailed namespace
    information).  If *events* is omitted, only ``"end"`` events are reported.
    *parser* is an optional parser instance.  If not given, the standard
    :class:`XMLParser` parser is used.  *parser* must be a subclass of
@@ -549,6 +550,10 @@ Functions
    .. deprecated:: 3.4
       The *parser* argument.
 
+   .. versionchanged:: 3.8
+      The ``comment`` and ``pi`` events were added.
+
+
 .. function:: parse(source, parser=None)
 
    Parses an XML section into an element tree.  *source* is a filename or file
@@ -1021,14 +1026,24 @@ TreeBuilder Objects
 ^^^^^^^^^^^^^^^^^^^
 
 
-.. class:: TreeBuilder(element_factory=None)
+.. class:: TreeBuilder(element_factory=None, comment_factory=None, \
+                       pi_factory=None)
 
    Generic element structure builder.  This builder converts a sequence of
    start, data, and end method calls to a well-formed element structure.  You
    can use this class to build an element structure using a custom XML parser,
-   or a parser for some other XML-like format.  *element_factory*, when given,
-   must be a callable accepting two positional arguments: a tag and
-   a dict of attributes.  It is expected to return a new element instance.
+   or a parser for some other XML-like format.
+
+   *element_factory*, when given, must be a callable accepting two positional
+   arguments: a tag and a dict of attributes.  It is expected to return a new
+   element instance.
+
+   The *comment_factory* and *pi_factory* functions, when given, should behave
+   like the :func:`Comment` and :func:`ProcessingInstruction` functions to
+   create comments and processing instructions.  When not given, no comments
+   or processing instructions will be created.  Note that these objects will
+   not currently be appended to the tree when they appear outside of the root
+   element.
 
    .. method:: close()
 
@@ -1053,6 +1068,21 @@ TreeBuilder Objects
       Opens a new element.  *tag* is the element name.  *attrs* is a dictionary
       containing element attributes.  Returns the opened element.
 
+   .. method:: comment(text)
+
+      Adds a comment with the given *text*.  If *comment_factory* is
+      :const:`None`, this will just return the text.
+
+      .. versionadded:: 3.8
+
+   .. method:: pi(target, text)
+
+      Adds a comment with the given *target* name and *text*.  If
+      *pi_factory* is :const:`None`, this will return a ``(target, text)``
+      tuple.
+
+      .. versionadded:: 3.8
+
 
    In addition, a custom :class:`TreeBuilder` object can provide the
    following method:
@@ -1150,9 +1180,9 @@ XMLPullParser Objects
    callback target, :class:`XMLPullParser` collects an internal list of parsing
    events and lets the user read from it. *events* is a sequence of events to
    report back.  The supported events are the strings ``"start"``, ``"end"``,
-   ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed
-   namespace information).  If *events* is omitted, only ``"end"`` events are
-   reported.
+   ``"comment"``, ``"pi"``, ``"start-ns"`` and ``"end-ns"`` (the "ns" events
+   are used to get detailed namespace information).  If *events* is omitted,
+   only ``"end"`` events are reported.
 
    .. method:: feed(data)
 
@@ -1172,6 +1202,10 @@ XMLPullParser Objects
       parser.  The iterator yields ``(event, elem)`` pairs, where *event* is a
       string representing the type of event (e.g. ``"end"``) and *elem* is the
       encountered :class:`Element` object.
+      For ``start-ns`` events, the ``elem`` is a tuple ``(prefix, uri)`` naming
+      the declared namespace mapping.  For ``end-ns`` events, the ``elem`` is
+      :const:`None`.  For ``comment`` events, the second value is the comment
+      text and for ``pi`` events a tuple ``(target, text)``.
 
       Events provided in a previous call to :meth:`read_events` will not be
       yielded again.  Events are consumed from the internal queue only when
@@ -1191,6 +1225,10 @@ XMLPullParser Objects
 
    .. versionadded:: 3.4
 
+   .. versionchanged:: 3.8
+      The ``comment`` and ``pi`` events were added.
+
+
 Exceptions
 ^^^^^^^^^^
 
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
@@ -1193,6 +1193,9 @@ def _feed(self, parser, data, chunk_size=None):
             for i in range(0, len(data), chunk_size):
                 parser.feed(data[i:i+chunk_size])
 
+    def assert_events(self, parser, expected):
+        self.assertEqual(list(parser.read_events()), expected)
+
     def assert_event_tags(self, parser, expected):
         events = parser.read_events()
         self.assertEqual([(action, elem.tag) for action, elem in events],
@@ -1275,8 +1278,10 @@ def test_events(self):
         self.assert_event_tags(parser, [])
 
         parser = ET.XMLPullParser(events=('start', 'end'))
-        self._feed(parser, "<!-- comment -->\n")
-        self.assert_event_tags(parser, [])
+        self._feed(parser, "<!-- text here -->\n")
+        self.assert_events(parser, [])
+
+        parser = ET.XMLPullParser(events=('start', 'end'))
         self._feed(parser, "<root>\n")
         self.assert_event_tags(parser, [('start', 'root')])
         self._feed(parser, "<element key='value'>text</element")
@@ -1313,6 +1318,34 @@ def test_events(self):
         self._feed(parser, "</root>")
         self.assertIsNone(parser.close())
 
+    def test_events_comment(self):
+        parser = ET.XMLPullParser(events=('start', 'comment', 'end'))
+        self._feed(parser, "<!-- text here -->\n")
+        self.assert_events(parser, [('comment', ' text here ')])
+        self._feed(parser, "<!-- more text here -->\n")
+        self.assert_events(parser, [('comment', ' more text here ')])
+        self._feed(parser, "<root-tag>text")
+        self.assert_event_tags(parser, [('start', 'root-tag')])
+        self._feed(parser, "<!-- inner comment-->\n")
+        self.assert_events(parser, [('comment', ' inner comment')])
+        self._feed(parser, "</root-tag>\n")
+        self.assert_event_tags(parser, [('end', 'root-tag')])
+        self._feed(parser, "<!-- outer comment -->\n")
+        self.assert_events(parser, [('comment', ' outer comment ')])
+
+        parser = ET.XMLPullParser(events=('comment',))
+        self._feed(parser, "<!-- text here -->\n")
+        self.assert_events(parser, [('comment', ' text here ')])
+
+    def test_events_pi(self):
+        parser = ET.XMLPullParser(events=('start', 'pi', 'end'))
+        self._feed(parser, "<?pitarget?>\n")
+        self.assert_events(parser, [('pi', ('pitarget', ''))])
+        parser = ET.XMLPullParser(events=('pi',))
+        self._feed(parser, "<?pitarget some text ?>\n")
+        self.assert_events(parser, [('pi', ('pitarget', 'some text '))])
+
+
     def test_events_sequence(self):
         # Test that events can be some sequence that's not just a tuple or list
         eventset = {'end', 'start'}
@@ -2658,6 +2691,31 @@ class DummyBuilder(BaseDummyBuilder):
         parser.feed(self.sample1)
         self.assertIsNone(parser.close())
 
+    def test_treebuilder_comment(self):
+        b = ET.TreeBuilder()
+        self.assertEqual(b.comment('ctext'), 'ctext')
+
+        b = ET.TreeBuilder(comment_factory=ET.Comment)
+        self.assertEqual(b.comment('ctext').tag, ET.Comment)
+        self.assertEqual(b.comment('ctext').text, 'ctext')
+
+        b = ET.TreeBuilder(comment_factory=len)
+        self.assertEqual(b.comment('ctext'), len('ctext'))
+
+    def test_treebuilder_pi(self):
+        b = ET.TreeBuilder()
+        self.assertEqual(b.pi('target', None), ('target', None))
+
+        b = ET.TreeBuilder(pi_factory=ET.PI)
+        self.assertEqual(b.pi('target').tag, ET.PI)
+        self.assertEqual(b.pi('target').text, "target")
+        self.assertEqual(b.pi('pitarget', ' text ').tag, ET.PI)
+        self.assertEqual(b.pi('pitarget', ' text ').text, "pitarget  text ")
+
+        b = ET.TreeBuilder(pi_factory=lambda target, text: (len(target), text))
+        self.assertEqual(b.pi('target'), (len('target'), None))
+        self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text '))
+
     def test_treebuilder_elementfactory_none(self):
         parser = ET.XMLParser(target=ET.TreeBuilder(element_factory=None))
         parser.feed(self.sample1)
@@ -2678,6 +2736,21 @@ def foobar(self, x):
         e = parser.close()
         self._check_sample1_element(e)
 
+    def test_subclass_comment_pi(self):
+        class MyTreeBuilder(ET.TreeBuilder):
+            def foobar(self, x):
+                return x * 2
+
+        tb = MyTreeBuilder(comment_factory=ET.Comment, pi_factory=ET.PI)
+        self.assertEqual(tb.foobar(10), 20)
+
+        parser = ET.XMLParser(target=tb)
+        parser.feed(self.sample1)
+        parser.feed('<!-- a comment--><?and a pi?>')
+
+        e = parser.close()
+        self._check_sample1_element(e)
+
     def test_element_factory(self):
         lst = []
         def myfactory(tag, attrib):
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
@@ -1374,21 +1374,31 @@ class TreeBuilder:
     *element_factory* is an optional element factory which is called
     to create new Element instances, as necessary.
 
+    *comment_factory* is a factory to create comments. If not provided,
+    comments will not be inserted into the tree and "comment" pull parser
+    events will only return the plain text.
+
+    *pi_factory* is a factory to create processing instructions. If not
+    provided, PIs will not be inserted into the tree and "pi" pull parser
+    events will only return a (target, text) tuple.
     """
-    def __init__(self, element_factory=None):
+    def __init__(self, element_factory=None, comment_factory=None, pi_factory=None):
         self._data = [] # data collector
         self._elem = [] # element stack
         self._last = None # last element
+        self._root = None # root element
         self._tail = None # true if we're after an end tag
+        self._comment_factory = comment_factory
+        self._pi_factory = pi_factory
         if element_factory is None:
             element_factory = Element
         self._factory = element_factory
 
     def close(self):
         """Flush builder buffers and return toplevel document Element."""
         assert len(self._elem) == 0, "missing end tags"
-        assert self._last is not None, "missing toplevel element"
-        return self._last
+        assert self._root is not None, "missing toplevel element"
+        return self._root
 
     def _flush(self):
         if self._data:
@@ -1417,6 +1427,8 @@ def start(self, tag, attrs):
         self._last = elem = self._factory(tag, attrs)
         if self._elem:
             self._elem[-1].append(elem)
+        elif self._root is None:
+            self._root = elem
         self._elem.append(elem)
         self._tail = 0
         return elem
@@ -1435,6 +1447,39 @@ def end(self, tag):
         self._tail = 1
         return self._last
 
+    def comment(self, text):
+        """Create a comment using the comment_factory.
+
+        If no factory is provided, comments are ignored
+        and the text returned as is.
+
+        *text* is the text of the comment.
+        """
+        if self._comment_factory is None:
+            return text
+        return self._handle_single(self._comment_factory, text)
+
+    def pi(self, target, text=None):
+        """Create a processing instruction using the pi_factory.
+
+        If no factory is provided, PIs are ignored and a (target, text)
+        tuple is returned.
+
+        *target* is the target name of the processing instruction.
+        *text* is the data of the processing instruction, or ''.
+        """
+        if self._pi_factory is None:
+            return (target, text)
+        return self._handle_single(self._pi_factory, target, text)
+
+    def _handle_single(self, factory, *args):
+        self._flush()
+        self._last = elem = factory(*args)
+        if self._elem:
+            self._elem[-1].append(elem)
+        self._tail = 1
+        return elem
+
 
 # also see ElementTree and TreeBuilder
 class XMLParser:
@@ -1519,6 +1564,15 @@ def handler(prefix, uri, event=event_name, append=append):
                 def handler(prefix, event=event_name, append=append):
                     append((event, None))
                 parser.EndNamespaceDeclHandler = handler
+            elif event_name == 'comment':
+                def handler(text, event=event_name, append=append, self=self):
+                    append((event, self.target.comment(text)))
+                parser.CommentHandler = handler
+            elif event_name == 'pi':
+                def handler(pi_target, data, event=event_name, append=append,
+                            self=self):
+                    append((event, self.target.pi(pi_target, data)))
+                parser.ProcessingInstructionHandler = handler
             else:
                 raise ValueError("unknown event %r" % event_name)
 
diff --git a/Misc/NEWS.d/next/Library/2019-04-20-09-50-32.bpo-36673.XF4Egb.rst b/Misc/NEWS.d/next/Library/2019-04-20-09-50-32.bpo-36673.XF4Egb.rst
@@ -0,0 +1,3 @@
+The TreeBuilder and XMLPullParser in xml.etree.ElementTree gained support
+for parsing comments and processing instructions.
+Patch by Stefan Behnel.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+The TreeBuilder and XMLPullParser in xml.etree.ElementTree gained support`
	`2`	`+for parsing comments and processing instructions.`
	`3`	`+Patch by Stefan Behnel.`