SL-19707 - use python3 translate functionality for escaping strings

Author: roxanneskelly

llsd/serde_xml.py

@@ -5,12 +5,24 @@
 
 from llsd.base import (_LLSD, ALL_CHARS, LLSDBaseParser, LLSDBaseFormatter, XML_HEADER,
                        LLSDParseError, LLSDSerializationError, UnicodeType,
-                       _format_datestr, _str_to_bytes, _to_python, is_unicode)
+                       _format_datestr, _str_to_bytes, _to_python, is_unicode, PY2)
 from llsd.fastest_elementtree import ElementTreeError, fromstring, parse as _parse
 
 INVALID_XML_BYTES = b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c'\
                     b'\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18'\
                     b'\x19\x1a\x1b\x1c\x1d\x1e\x1f'
+
+XML_ESC_TRANS = {}
+if not PY2:
+    XML_ESC_TRANS = str.maketrans({'&': '&',
+                                   '<':'&lt;',
+                                   '>':'&gt;',
+                                   u'\uffff':None,   # cannot be parsed
+                                   u'\ufffe':None})  # cannot be parsed
+
+    for x in INVALID_XML_BYTES:
+        XML_ESC_TRANS[x] = None
+
 INVALID_XML_RE = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
 
 
@@ -25,6 +37,24 @@ def remove_invalid_xml_bytes(b):
         # unit tests)
         return INVALID_XML_RE.sub('', b)
 
+def xml_esc(v):
+    "Escape string or unicode object v for xml output"
+
+    # Use is_unicode() instead of is_string() because in python 2, str is
+    # bytes, not unicode, and should not be "encode()"d. Attempts to
+    # encode("utf-8") a bytes type will result in an implicit
+    # decode("ascii") that will throw a UnicodeDecodeError if the string
+    # contains non-ascii characters.
+    if is_unicode(v):
+        # we need to drop these invalid characters because they
+        # cannot be parsed (and encode() doesn't drop them for us)
+        v = v.replace(u'\uffff', u'')
+        v = v.replace(u'\ufffe', u'')
+        v = v.encode('utf-8')
+    v = remove_invalid_xml_bytes(v)
+    return v.replace(b'&',b'&amp;').replace(b'<',b'&lt;').replace(b'>',b'&gt;')
+
+
 
 class LLSDXMLFormatter(LLSDBaseFormatter):
     """
@@ -38,47 +68,41 @@ class LLSDXMLFormatter(LLSDBaseFormatter):
     interface to this functionality.
     """
 
-    def xml_esc(self, v):
-        "Escape string or unicode object v for xml output"
-
-        # Use is_unicode() instead of is_string() because in python 2, str is
-        # bytes, not unicode, and should not be "encode()"d. Attempts to
-        # encode("utf-8") a bytes type will result in an implicit
-        # decode("ascii") that will throw a UnicodeDecodeError if the string
-        # contains non-ascii characters.
-        if is_unicode(v):
-            # we need to drop these invalid characters because they
-            # cannot be parsed (and encode() doesn't drop them for us)
-            v = v.replace(u'\uffff', u'')
-            v = v.replace(u'\ufffe', u'')
-            v = v.encode('utf-8')
-        v = remove_invalid_xml_bytes(v)
-        return v.replace(b'&',b'&amp;').replace(b'<',b'&lt;').replace(b'>',b'&gt;')
-
+    def __init__(self, indent_atom = None):
+        "Construct a pretty serializer."
+        # Call the super class constructor so that we have the type map
+        super(LLSDXMLFormatter, self).__init__()
+        self.py2 = PY2
+        
     def _LLSD(self, v):
         raise LLSDSerializationError("We should never end up here")
     def _UNDEF(self, _v):
         return b'<undef/>'
     def _BOOLEAN(self, v):
         if v:
             return b'<boolean>true</boolean>'
-        else:
-            return b'<boolean>false</boolean>'
+        return b'<boolean>false</boolean>'
     def _INTEGER(self, v):
-        return b'<integer>' + str(v).encode() + b'</integer>'
+        return b'<integer>' + str(v).encode('utf-8') + b'</integer>'
     def _REAL(self, v):
-        return b'<real>' + str(v).encode() + b'</real>'
+        return b'<real>' + str(v).encode('utf-8') + b'</real>'
     def _UUID(self, v):
         if v.int == 0:
             return b'<uuid/>'
         else:
-            return b'<uuid>' + str(v).encode() + b'</uuid>'
+            return b'<uuid>' + str(v).encode('utf-8') + b'</uuid>'
     def _BINARY(self, v):
         return b'<binary>' + base64.b64encode(v).strip() + b'</binary>'
     def _STRING(self, v):
-        return b'<string>' + self.xml_esc(v) + b'</string>'
+        if self.py2:
+            return b'<string>' + _str_to_bytes(xml_esc(v)) + b'</string>'
+        else:
+            return b'<string>' + v.translate(XML_ESC_TRANS).encode('utf-8') + b'</string>'
     def _URI(self, v):
-        return b'<uri>' + self.xml_esc(v) + b'</uri>'
+        if self.py2:
+            return b'<uri>' + _str_to_bytes(xml_esc(v)) + b'</uri>'
+        else:
+            return b'<uri>' + UnicodeType(v).translate(XML_ESC_TRANS).encode('utf-8') + b'</uri>'
     def _DATE(self, v):
         return b'<date>' + _format_datestr(v) + b'</date>'
     def _ARRAY(self, v):
@@ -97,28 +121,38 @@ def _write(self, something):
 
         iter_stack = [(iter([something]), b"", None)]
         while True:
-            cur_iter, iter_type, iterable = iter_stack[-1]
+            cur_iter, iter_type, iterable_obj = iter_stack[-1]
             try:
                 item = next(cur_iter)
                 if iter_type == b"map":
-                    self.stream.write(b'<key>' + self.xml_esc(UnicodeType(item)) + b'</key>')
-                    item = iterable[item]
+
+                    if self.py2:
+                        self.stream.write(b'<key>' +
+                                          _str_to_bytes(xml_esc(UnicodeType(item))) +
+                                          b'</key>')
+                    else:
+                        # fair performance improvement by explicitly doing the
+                        # translate for py3 instead of calling xml_esc
+                        self.stream.write(b'<key>' +
+                        UnicodeType(item).translate(XML_ESC_TRANS).encode('utf-8') +
+                                          b'</key>')
+                    item = iterable_obj[item]
                 if isinstance(item, _LLSD):
                     item = item.thing
-                t = type(item)
-                if not t in self.type_map:
+                item_type = type(item)
+                if not item_type in self.type_map:
                     raise LLSDSerializationError(
-                        "Cannot serialize unknown type: %s (%s)" % (t, item))
-                tf = self.type_map[t]
+                        "Cannot serialize unknown type: %s (%s)" % (item_type, item))
+                tfunction = self.type_map[item_type]
 
-                if tf == self._MAP:
+                if tfunction == self._MAP:
                     self.stream.write(b'<map>')
                     iter_stack.append((iter(list(item)), b"map", item))
-                elif tf == self._ARRAY:
+                elif tfunction == self._ARRAY:
                     self.stream.write(b'<array>')
                     iter_stack.append((iter(item), b"array", None))
                 else:
-                    self.stream.write(tf(item))
+                    self.stream.write(tfunction(item))
             except StopIteration:
                 self.stream.write(b'</' + iter_type + b'>')
                 iter_stack.pop()
@@ -156,14 +190,6 @@ def _indent(self):
         "Write an indentation based on the atom and indentation level."
         self.stream.writelines([self._indent_atom] * self._indent_level)
 
-    def _ARRAY(self, v):
-        "Recursively format an array with pretty turned on."
-        raise LLSDSerializationError("We should never end up here")
-
-    def _MAP(self, v):
-        "Recursively format a map with pretty turned on."
-        raise LLSDSerializationError("We should never end up here")
-
     def _write(self, something):
         """
         Serialize a python object to self.stream as application/llsd+xml.
@@ -178,36 +204,44 @@ def _write(self, something):
         self.stream.write(b'<?xml version="1.0" ?>\n'
                           b'<llsd>\n')
 
-        iter_stack = [(iter([something]), b"")]
+        iter_stack = [(iter([something]), b"", None)]
         while True:
-            cur_iter, iter_type = iter_stack[-1]
+            cur_iter, iter_type, iterable_obj = iter_stack[-1]
             try:
                 item = next(cur_iter)
                 if iter_type == b"map":
                     self._indent()
-                    self.stream.write(b'<key>' + _str_to_bytes(self.xml_esc(UnicodeType(item[0]))) + b'</key>\n')
-                    item = item[1]
+                    if self.py2:
+                        self.stream.write(b'<key>' +
+                                          _str_to_bytes(xml_esc(UnicodeType(item))) +
+                                          b'</key>')
+                    else:
+                        # calling translate directly is a bit faster
+                        self.stream.write(b'<key>' +
+                        UnicodeType(item).translate(XML_ESC_TRANS).encode('utf-8') +
+                                          b'</key>\n')
+                    item = iterable_obj[item]
                 if isinstance(item, _LLSD):
                     item = item.thing
-                t = type(item)
-                if not t in self.type_map:
+                item_type = type(item)
+                if not item_type in self.type_map:
                     raise LLSDSerializationError(
-                        "Cannot serialize unknown type: %s (%s)" % (t, item))
-                tf = self.type_map[t]
+                        "Cannot serialize unknown type: %s (%s)" % (item_type, item))
+                tfunction = self.type_map[item_type]
 
-                if tf == self._MAP:
+                if tfunction == self._MAP:
                     self._indent()
                     self.stream.write(b'<map>\n')
                     self._indent_level += 1
-                    iter_stack.append((iter(item.items()), b"map"))
-                elif tf == self._ARRAY:
+                    iter_stack.append((iter(list(item)), b"map", item))
+                elif tfunction == self._ARRAY:
                     self._indent()
                     self.stream.write(b'<array>\n')
                     self._indent_level += 1
-                    iter_stack.append((iter(item), b"array"))
+                    iter_stack.append((iter(item), b"array", None))
                 else:
                     self._indent()
-                    self.stream.write(tf(item))
+                    self.stream.write(tfunction(item))
                     self.stream.write(b'\n')
             except StopIteration:
                 self._indent_level -= 1

tests/bench.py

@@ -82,7 +82,6 @@ def binary_stream():
         yield f
 
 def build_deep_xml():
-
     deep_data = {}
     curr_data = deep_data
     for i in range(250):
@@ -96,16 +95,14 @@ def build_deep_xml():
 _deep_bench_data = build_deep_xml()
 
 def build_wide_xml():
+    
     wide_xml = b"""
 <?xml version="1.0" encoding="UTF-8"?><llsd><map><key>wide_array</key><array>"
 """
-
+    wide_data = {}
     for i in range(100000):
-        wide_xml += b"""
-    <real>5000</real>"""
-    wide_xml += b"</array></map></llsd>"
-
-    return llsd.parse_xml(wide_xml)
+        wide_data["item"+str(i)] = {"item1":2.345, "item2": [1,2,3], "item3": "string", "item4":{"subitem": llsd.uri("http://foo.bar.com")}}
+    return wide_data
 _wide_bench_data = build_wide_xml()
 
 def bench_stream(parse, stream):