Ticket #129: large-logfile-crash-fix.patch

==== //depot/Demitech/BuildSlave/buildbot/slave/commands.py#2 (text) ====

@@ -211,7 +211,8 @@
     notreally = False
     BACKUP_TIMEOUT = 5
     KILL = "KILL"
-
+    CHUNK_LIMIT = 128*1024
+    
     def __init__(self, builder, command,
                  workdir, environ=None,
                  sendStdout=True, sendStderr=True, sendRC=True,
@@ -284,6 +285,14 @@
     def sendStatus(self, status):
         self.builder.sendUpdate(status)
 
+    # twisted current craps out if we stuff more than 640k of data through its serialization -- so
+    # we have to chunk data up if we're reading it from potentially large stdio reads
+    def _chunkForSend(self, data):
+       while length(data) > ShellCommand.CHUNK_SIZE:
+           yield data[:ShellCommand.CHUNK_SIZE]
+           data = data[ShellCommand.CHUNK_SIZE:]
+       yield data
+       
     def start(self):
         # return a Deferred which fires (with the exit code) when the command
         # completes
@@ -393,7 +402,8 @@
 
     def addStdout(self, data):
         if self.sendStdout:
-            self.sendStatus({'stdout': data})
+           for chunk in self._chunkForSend(data):
+               self.sendStatus({'stdout': chunk})
         if self.keepStdout:
             self.stdout += data
         if self.timer:
@@ -401,12 +411,14 @@
 
     def addStderr(self, data):
         if self.sendStderr:
-            self.sendStatus({'stderr': data})
+           for chunk in self._chunkForSend(data):
+               self.sendStatus({'stderr': chunk})
         if self.timer:
             self.timer.reset(self.timeout)
 
     def addLogfile(self, name, data):
-        self.sendStatus({'log': (name, data)})
+       for chunk in self._chunkForSend(data):
+            self.sendStatus({'log': (name, chunk)})
         if self.timer:
             self.timer.reset(self.timeout)