PyLint command implementation similar to the PyFlakes one

[strank]

Here's a PyLint? command I use in a master.cfg, maybe it's useful for inclusion in buildbot?

from buildbot.steps.shell import ShellCommand
from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS
import re
try:
    import cStringIO
    StringIO = cStringIO.StringIO
except ImportError:
    from StringIO import StringIO

class PyLint(ShellCommand):
    '''A command that knows about pylint output.
    It's a good idea to add --output-format=parseable to your
    command, since it includes the filename in the message.
    '''
    name = "pylint"
    description = ["running", "pylint"]
    descriptionDone = ["pylint"]

    # Using the default text output, the message format is :
    # MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE
    # with --output-format=parseable it is: (the outer brackets are literal)
    # FILE_NAME:LINE_NUM: [MESSAGE_TYPE[, OBJECT]] MESSAGE
    # message type consists of the type char and 4 digits
    # The message types:

    MESSAGES = {
            'C': "convention", # for programming standard violation
            'R': "refactor", # for bad code smell
            'W': "warning", # for python specific problems
            'E': "error", # for much probably bugs in the code
            'F': "fatal", # error prevented pylint from further processing.
            'I': "info",
        }

    flunkingIssues = ["F", "E"] # msg categories that cause FAILURE

    _re_groupname = 'errtype'
    _msgtypes_re_str = '(?P<%s>[%s])' % (_re_groupname, ''.join(MESSAGES.keys()))
    _default_line_re = re.compile(r'%s\d{4}: *\d+:.+' % _msgtypes_re_str)
    _parseable_line_re = re.compile(r'[^:]+:\d+: \[%s\d{4}[,\]] .+' % _msgtypes_re_str)

    def createSummary(self, log):
        counts = {}
        summaries = {}
        for m in self.MESSAGES:
            counts[m] = 0
            summaries[m] = []

        line_re = None # decide after first match
        for line in StringIO(log.getText()).readlines():
            if not line_re:
                # need to test both and then decide on one
                if self._parseable_line_re.match(line):
                    line_re = self._parseable_line_re
                elif self._default_line_re.match(line):
                    line_re = self._default_line_re
                else: # no match yet
                    continue
            mo = line_re.match(line)
            if mo:
                msgtype = mo.group(self._re_groupname)
                assert msgtype in self.MESSAGES
            summaries[msgtype].append(line)
            counts[msgtype] += 1

        self.descriptionDone = self.descriptionDone[:]
        for msg, fullmsg in self.MESSAGES.items():
            if counts[msg]:
                self.descriptionDone.append("%s=%d" % (fullmsg, counts[msg]))
                self.addCompleteLog(fullmsg, "".join(summaries[msg]))
            self.setProperty("pylint-%s" % fullmsg, counts[msg])
        self.setProperty("pylint-total", sum(counts.values()))

    def evaluateCommand(self, cmd):
        if cmd.rc != 0:
            return FAILURE
        for msg in self.flunkingIssues:
            if self.getProperty("pylint-%s" % self.MESSAGES[msg]):
                return FAILURE
        if self.getProperty("pylint-total"):
            return WARNINGS
        return SUCCESS

cheers

[mue]

I would like a pylint-step very much. Currently I'm trying the code above. To make pylint check the files from the change which trigged the build, I've made the following extensions:

--- buildbot/process/base.py.old        2008-09-16 18:02:34.000000000 +0200
+++ buildbot/process/base.py    2008-11-18 10:33:52.000000000 +0100
@@ -282,6 +282,8 @@
         props.setProperty("buildnumber", self.build_status.number, "Build")
         props.setProperty("branch", self.source.branch, "Build")
         props.setProperty("revision", self.source.revision, "Build")
+       # for pylint-step
+       props.setProperty("files", self.allFiles(), "Build")

     def setupSlaveBuilder(self, slavebuilder):
         self.slavebuilder = slavebuilder

And to the PyLint? class:

    def start(self):
        # this block is specific to ShellCommands. subclasses that don't need
        # to set up an argv array, an environment, or extra logfiles= (like
        # the Source subclasses) can just skip straight to startCommand()
        properties = self.build.getProperties()

        # create the actual RemoteShellCommand instance now
        kwargs = properties.render(self.remote_kwargs)
        filesToCheck = properties.getProperty("files")
        if not filesToCheck:
            log.msg("No files to check")
            return FAILURE
        command = properties.render(self.command)
        command.extend(filesToCheck)
        log.msg("DEBUG: %s" % str(command))
        kwargs['command'] = command
        kwargs['logfiles'] = self.logfiles
        cmd = RemoteShellCommand(**kwargs)
        self.setupEnvironment(cmd)
        self.checkForOldSlaveAndLogfiles()
        self.cmd = cmd

        self.startCommand(cmd)

At the moment I have some trouble to parse the log, but very likely this is my fault.

[dustin]

I like the OP's version.

[dustin]

commit 620c2487bddc693a52368d4c9dcb5fd37e995a6c
Author: Dustin J. Mitchell <dustin@zmanda.com>
Date:   Wed Feb 18 16:23:29 2009 -0500

    (refs #259) add docs for PyLint step

commit da4954b47a89a95ace7a6896c259ad4f812e568c
Author: strank <list-ener@strank.info>
Date:   Wed Feb 18 16:21:04 2009 -0500

    (refs #259) PyLink command, similar to PyFlakes