Merge pull request #6 from secondlife/sl-18330-perf

SL-18330: Refactor notation parsing to manage a lookahead char.

Author: nat-goodspeed

llsd/base.py

@@ -404,13 +404,12 @@ def _reset(self, something):
             # Wrap an incoming bytes string into a stream. If the passed bytes
             # string is so large that the overhead of copying it into a
             # BytesIO is significant, advise caller to pass a stream instead.
-            # BytesIO has no peek() method, so wrap it in BufferedReader.
-            self._stream = io.BufferedReader(io.BytesIO(something))
-        elif hasattr(something, 'peek'):
-            # 'something' is already a buffered stream, use directly
+            self._stream = io.BytesIO(something)
+        elif something.seekable():
+            # 'something' is already a seekable stream, use directly
             self._stream = something
         else:
-            # 'something' isn't buffered, wrap in BufferedReader
+            # 'something' isn't seekable, wrap in BufferedReader
             # (let BufferedReader handle the problem of passing an
             # inappropriate object)
             self._stream = io.BufferedReader(something)
@@ -482,49 +481,23 @@ def _next_nonblank(self):
             c = self._stream.read(1)
         return c
 
-    def _getc(self, num=1):
+    def _getc(self, num=1, full=True):
         got = self._stream.read(num)
-        if len(got) < num:
+        if full and len(got) < num:
             self._error("Trying to read past end of stream")
         return got
 
-    def _peek(self, num=1, full=True):
-        # full=True means error if we can't peek ahead num bytes
-        if num < 0:
-            # There aren't many ways this can happen. The likeliest is that
-            # we've just read garbage length bytes from a binary input string.
-            # We happen to know that lengths are encoded as 4 bytes, so back
-            # off by 4 bytes to try to point the user at the right spot.
-            self._error("Invalid length field %d" % num, -4)
-
-        # Instead of using self._stream.peek() at all, use read(num) and reset
-        # the read pointer. BufferedReader.peek() does not promise to return
-        # the requested length, but does not clarify the conditions under
-        # which it returns fewer bytes.
-        # https://docs.python.org/3/library/io.html#io.BufferedReader.peek
-        # In practice, we've seen it fail with an input file up over 100Kb:
-        # peek() returns only part of what we requested, but because we also
-        # passed full=False (see LLSDNotationParser._get_re()), we didn't
-        # notice and the parse failed looking for a map delimiter halfway
-        # through a large decimal integer. read(num), on the other hand,
-        # promises to return num bytes until actual EOF.
-        oldpos = self._stream.tell()
-        try:
-            got = self._stream.read(num)
-            if full and len(got) < num:
-                self._error("Trying to peek past end of stream")
-
-            return got
-
-        finally:
-            self._stream.seek(oldpos)
+    def _putback(self, cc):
+        # if this test fails, it's not a user error, it's a coding error
+        assert self._stream.tell() >= len(cc)
+        self._stream.seek(-len(cc), io.SEEK_CUR)
 
     def _error(self, message, offset=0):
         oldpos = self._stream.tell()
         # 'offset' is relative to current pos
         self._stream.seek(offset, io.SEEK_CUR)
         raise LLSDParseError("%s at byte %d: %r" %
-                             (message, oldpos+offset, self._peek(1, full=False)))
+                             (message, oldpos+offset, self._getc(1, full=False)))
 
     # map char following escape char to corresponding character
     _escaped = {

llsd/serde_binary.py

@@ -8,6 +8,14 @@
                        _str_to_bytes, binary, is_integer, is_string, uri)
 
 
+try:
+    # Python 2: make 'range()' lazy like Python 3
+    range = xrange
+except NameError:
+    # Python 3: 'range()' is already lazy
+    pass
+
+
 class LLSDBinaryParser(LLSDBaseParser):
     """
     Parse application/llsd+binary to a python object.
@@ -56,15 +64,16 @@ def __init__(self):
         for c, func in _dispatch_dict.items():
             self._dispatch[ord(c)] = func
 
-    def parse(self, buffer, ignore_binary = False):
+    def parse(self, something, ignore_binary = False):
         """
         This is the basic public interface for parsing.
 
-        :param buffer: the binary data to parse in an indexable sequence.
+        :param something: serialized LLSD to parse: a bytes object, a binary
+                          stream or an LLSDBaseParser subclass.
         :param ignore_binary: parser throws away data in llsd binary nodes.
         :returns: returns a python object.
         """
-        self._reset(buffer)
+        self._reset(something)
         self._keep_binary = not ignore_binary
         try:
             return self._parse()
@@ -107,15 +116,10 @@ def _parse_array(self):
         "Parse a single llsd array"
         rv = []
         size = struct.unpack("!i", self._getc(4))[0]
-        count = 0
-        cc = self._peek()
-        while (cc != b']') and (count < size):
+        for count in range(size):
             rv.append(self._parse())
-            count += 1
-            cc = self._peek()
-        if cc != b']':
+        if self._getc() != b']':
             self._error("invalid array close token")
-        self._getc()
         return rv
 
     def _parse_string(self):