Changeset 933 in tailor

Timestamp:

10/20/05 23:33:41 (8 years ago)

Author:

lele@…

Hash name:

20051020213341-7a6fb-a00122f770aab206ad9d16f7fd93e7c7870bc565

Message:

Use an incremental parser for svn log
This transforms the changesets_from_svnlog() function into a generator
that yields changesets as they are parsed out from XML, that is read
in chunks.

Location:

vcpx

Files:

• svn.py (modified) (2 diffs)
• tests/svn.py (modified) (14 diffs)

vcpx/svn.py

@@ -18 +18 @@
 from config import ConfigurationError
 
-def changesets_from_svnlog(log, repository, module):
-    from xml.sax import parse
-    from xml.sax.handler import ContentHandler
+def changesets_from_svnlog(log, repository, module, chunksize=2**15):
+    from xml.sax import make_parser
+    from xml.sax.handler import ContentHandler, ErrorHandler
     from changes import ChangesetEntry, Changeset
     from datetime import datetime
@@ -183 +183 @@
             self.current_field.append(data)
 
+    parser = make_parser()
     handler = SvnXMLLogHandler()
-    parse(log, handler)
-    return handler.changesets
+    parser.setContentHandler(handler)
+    parser.setErrorHandler(ErrorHandler())
+
+    chunk = log.read(chunksize)
+    while chunk:
+        parser.feed(chunk)
+        for cs in handler.changesets:
+            yield cs
+        handler.changesets = []
+        chunk = log.read(chunksize)
+    parser.close()
+    for cs in handler.changesets:
+        yield cs
 
 

vcpx/tests/svn.py

@@ -25 +25 @@
         log = self.getSvnLog('svn-simple_rename_test')
         csets = changesets_from_svnlog(log, 'file:///tmp/t/repo', '/trunk')
-        self.assertEqual(len(csets), 2)
-
-        cset = csets[0]
+
+        cset = csets.next()
         self.assertEqual(cset.author, 'lele')
         self.assertEqual(cset.date, datetime(2004,11,12,15,05,37,134366))
@@ -41 +40 @@
         self.assertEqual(entry.action_kind, entry.ADDED)
 
-        cset = csets[1]
+        cset = csets.next()
         self.assertEqual(cset.author, 'lele')
         self.assertEqual(cset.date, datetime(2004,11,12,15,06,04,193650))
@@ -52 +51 @@
         self.assertEqual(entry.old_name, 'dir')
 
+        self.assertRaises(StopIteration, csets.next)
+
     def testRenameOutBehaviour(self):
         """Verify svn log parser behaves correctly on renames out of scope"""
@@ -57 +58 @@
         log = self.getSvnLog('svn-rename_out_test')
         csets = changesets_from_svnlog(log, 'http://srv/svn/Shtoom', '/trunk')
-        self.assertEqual(len(csets), 1)
-
-        cset = csets[0]
+
+        cset = csets.next()
         self.assertEqual(cset.author, 'anthony')
         self.assertEqual(cset.date, datetime(2004,11,9,6,54,20,709243))
@@ -69 +69 @@
         self.assertEqual(entry.action_kind, entry.DELETED)
 
+        self.assertRaises(StopIteration, csets.next)
+
     def testCopyAndRename(self):
         """Verify svn log parser behaves correctly on copies"""
 
         log = self.getSvnLog('svn-copy_and_rename_test')
-        csets = changesets_from_svnlog(log, 'file:///tmp/rep', '/test')
+        csets = list(changesets_from_svnlog(log, 'file:///tmp/rep', '/test'))
         self.assertEqual(len(csets), 4)
 
@@ -111 +113 @@
         log = self.getSvnLog('svn-svn_r_event_test')
         csets = changesets_from_svnlog(log, 'file:///tmp/rep', '/trunk')
-        self.assertEqual(len(csets), 2)
-
-        cset = csets[1]
+
+        cset = csets.next()
+
+        cset = csets.next()
         self.assertEqual(cset.author, 'cmlenz')
-        self.assertEqual(cset.date, datetime(2005,3,21, 8,34,02,522947))
+        self.assertEqual(cset.date, datetime(2005,3,21, 8,34, 2,522947))
         self.assertEqual(len(cset.entries), 7)
 
@@ -143 +146 @@
         self.assertEqual(entry.action_kind, entry.UPDATED)
 
+        self.assertRaises(StopIteration, csets.next)
+
     def testTrackingRoot(self):
         """Verify we are able to track the root of the repository"""
 
         log = self.getSvnLog('svn-svn_repos_root_test')
-        csets = changesets_from_svnlog(log, 'svn+ssh://caia/tmp/svn', '/')
+        csets = list(changesets_from_svnlog(log, 'svn+ssh://caia/tmp/svn', '/'))
         self.assertEqual(len(csets), 4)
 
@@ -171 +176 @@
         csets = changesets_from_svnlog(log, 'http://srv/svn', '/py/dist')
 
-        self.assertEqual(len(csets), 1)
-
-        cset = csets[0]
+        cset = csets.next()
         self.assertEqual(len(cset.entries), 3)
 
@@ -188 +191 @@
         self.assertEqual(entry.name, 'py/documentation/test.txt')
         self.assertEqual(entry.action_kind, entry.UPDATED)
+
+        self.assertRaises(StopIteration, csets.next)
 
     def testUnicode(self):
@@ -196 +201 @@
                                        '/branches/2.1')
 
-        self.assertEqual(len(csets), 1)
-
-        log = csets[0].log
+        log = csets.next().log
         self.assertEqual(type(log), type(u'€'))
         self.assertEqual(len(log), 91)
@@ -204 +207 @@
         self.assertEqual(len(log.encode('ascii', 'ignore')), 90)
 
+        self.assertRaises(StopIteration, csets.next)
+
     def testCopyAndReplace(self):
         """Verify the svn parser handle copy+replace"""
@@ -210 +215 @@
         csets = changesets_from_svnlog(log, 'http://srv/repos/trac', '/trunk')
 
-        self.assertEqual(len(csets), 1)
-
-        cset = csets[0]
+        cset = csets.next()
         self.assertEqual(len(cset.entries), 7)
 
@@ -243 +246 @@
                                        '/branches/SAMBA_4_0')
 
-        self.assertEqual(len(csets), 1)
-
-        cset = csets[0]
+        cset = csets.next()
         self.assertEqual(len(cset.entries), 3)
 
@@ -259 +260 @@
         self.assertEqual(entry.name, 'source/nsswitch/wb_common.c')
         self.assertEqual(entry.action_kind, entry.ADDED)
+
+    def testIncrementalParser(self):
+        """Verify that the svn log parser is effectively incremental"""
+
+        log = self.getSvnLog('svn-svn_repos_root_test')
+        csets = list(changesets_from_svnlog(log, 'svn+ssh://caia/tmp/svn', '/',
+                                            chunksize=100))
+        self.assertEqual(len(csets), 4)