Ticket #56: triggerable.patch

buildbot/scheduler.py

@@ -681 +681 @@
         from buildbot.status.client import makeRemote
         return makeRemote(bs.status)
 
+class Triggerable(BaseUpstreamScheduler):
+    """
+    This scheduler doesn't do anything until it is triggered by
+    a Trigger step in a factory.
+    """
+
+    def __init__(self, name, builderNames):
+        BaseUpstreamScheduler.__init__(self, name)
+        self.builderNames = builderNames
+
+    def listBuilderNames(self):
+        return self.builderNames
+
+    def getPendingBuildTimes(self):
+        return []
+
+    def trigger(self, ss):
+        """
+        Trigger this scheduler.  Returns a deferred that will fire when the buildset
+        is finished.
+        """
+        bs = buildset.BuildSet(self.builderNames, ss)
+        d = bs.waitUntilFinished()
+        self.submit(bs)
+        return d

buildbot/steps/trigger.py

@@ +1 @@
+from buildbot.process.buildstep import LoggingBuildStep, SUCCESS, FAILURE, EXCEPTION
+from buildbot.steps.shell import WithProperties
+from buildbot.sourcestamp import SourceStamp
+from buildbot.scheduler import Triggerable
+from twisted.internet import defer
+from twisted.python import log
+import os
+
+class Trigger(LoggingBuildStep):
+    """
+    I trigger a Triggerable.  It's fun.
+    """
+    name = "trigger"
+
+    flunkOnFailure = True
+
+    def __init__(self,
+        schedulers=[],
+        updateSourceStamp=False,
+        waitForFinish=False,
+        **kwargs):
+        """
+        Trigger the given schedulers when this step is executed.
+
+        @var schedulers: list of schedulers' names that should be triggered.  Schedulers
+        can be specified using WithProperties, if desired.
+
+        @var updateSourceStamp: should I update the source stamp to
+        an absolute SourceStamp before triggering a new build?
+
+        @var waitForFinish: should I wait for all of the triggered schedulers to finish
+        their builds?
+        """
+        assert schedulers, "You must specify a scheduler to trigger"
+        self.schedulers = schedulers
+        self.updateSourceStamp = updateSourceStamp
+        self.waitForFinish = waitForFinish
+        self.running = False
+        LoggingBuildStep.__init__(self, **kwargs)
+        self.addFactoryArguments(schedulers=schedulers,
+                                 updateSourceStamp=updateSourceStamp,
+                                 waitForFinish=waitForFinish)
+
+    def interrupt(self, reason):
+        if self.running:
+            self.step_status.setColor("red")
+            self.step_status.setText(["interrupted"])
+
+    def start(self):
+        self.running = True
+        ss = self.build.getSourceStamp()
+        if self.updateSourceStamp:
+            ss = ss.getAbsoluteSourceStamp(self.build.getProperty('got_revision'))
+        # (is there an easier way to find the BuildMaster?)
+        all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
+        all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
+        unknown_schedulers = []
+        triggered_schedulers = []
+
+        dl = []
+        for scheduler in self.schedulers:
+            if isinstance(scheduler, WithProperties):
+                scheduler = scheduler.render(self.build)
+            if all_schedulers.has_key(scheduler):
+                sch = all_schedulers[scheduler]
+                if isinstance(sch, Triggerable):
+                    dl.append(sch.trigger(ss))
+                    triggered_schedulers.append(scheduler)
+                else:
+                    unknown_schedulers.append(scheduler)
+            else:
+                unknown_schedulers.append(scheduler)
+
+        if unknown_schedulers:
+            self.step_status.setColor("red")
+            self.step_status.setText(['no scheduler:'] + unknown_schedulers)
+            rc = FAILURE
+        else:
+            rc = SUCCESS
+            self.step_status.setText(['triggered'] + triggered_schedulers)
+            if self.waitForFinish:
+                self.step_status.setColor("yellow")
+            else:
+                self.step_status.setColor("green")
+
+        if self.waitForFinish:
+            d = defer.DeferredList(dl, consumeErrors=1)
+        else:
+            d = defer.succeed([])
+
+        def cb(rclist):
+            rc = SUCCESS
+            for was_cb, buildsetstatus in rclist:
+                # TODO: make this algo more configurable
+                if not was_cb:
+                    rc = EXCEPTION
+                    break
+                if buildsetstatus.getResults() == FAILURE:
+                    rc = FAILURE
+            return self.finished(rc)
+
+        def eb(why):
+            return self.finished(FAILURE)
+
+        d.addCallbacks(cb, eb)

buildbot/test/test_run.py

@@ -628 +628 @@
         d = self.master.loadConfig(config_4_newbuilder)
         return d
 
+class Triggers(RunMixin, TestFlagMixin, unittest.TestCase):
+    config_trigger = config_base + """
+from buildbot.scheduler import Triggerable, Scheduler
+from buildbot.steps.trigger import Trigger
+from buildbot.steps.dummy import Dummy
+from buildbot.test.runutils import SetTestFlagStep
+c['schedulers'] = [
+    Scheduler('triggerer', None, 0.1, ['triggerer']),
+    Triggerable('triggeree', ['triggeree'])
+]
+triggerer = factory.BuildFactory([
+    s(SetTestFlagStep, flagname='triggerer_started'),
+    s(Trigger, flunkOnFailure=True, @ARGS@),
+    s(SetTestFlagStep, flagname='triggerer_finished'),
+    ])
+triggeree = factory.BuildFactory([
+    s(SetTestFlagStep, flagname='triggeree_started'),
+    s(@DUMMYCLASS@),
+    s(SetTestFlagStep, flagname='triggeree_finished'),
+    ])
+c['builders'] = [{'name': 'triggerer', 'slavename': 'bot1',
+                  'builddir': 'triggerer', 'factory': triggerer},
+                 {'name': 'triggeree', 'slavename': 'bot1',
+                  'builddir': 'triggeree', 'factory': triggeree}]
+"""
+
+    def mkConfig(self, args, dummyclass="Dummy"):
+        return self.config_trigger.replace("@ARGS@", args).replace("@DUMMYCLASS@", dummyclass)
+
+    def setupTest(self, args, dummyclass, checkFn):
+        self.clearFlags()
+        m = self.master
+        m.loadConfig(self.mkConfig(args, dummyclass))
+        m.readConfig = True
+        m.startService()
+
+        c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
+        m.change_svc.addChange(c)
+
+        d = self.connectSlave(builders=['triggerer', 'triggeree'])
+        d.addCallback(self.startTimer, 0.5, checkFn)
+        return d
+
+    def startTimer(self, res, time, next_fn):
+        d = defer.Deferred()
+        reactor.callLater(time, d.callback, None)
+        d.addCallback(next_fn)
+        return d
+
+    def testTriggerBuild(self):
+        return self.setupTest("schedulers=['triggeree']",
+                "Dummy",
+                self._checkTriggerBuild)
+
+    def _checkTriggerBuild(self, res):
+        self.failIfFlagNotSet('triggerer_started')
+        self.failIfFlagNotSet('triggeree_started')
+        self.failIfFlagSet('triggeree_finished')
+        self.failIfFlagNotSet('triggerer_finished')
+
+    def testTriggerBuildWait(self):
+        return self.setupTest("schedulers=['triggeree'], waitForFinish=1",
+                "Dummy",
+                self._checkTriggerBuildWait)
+
+    def _checkTriggerBuildWait(self, res):
+        self.failIfFlagNotSet('triggerer_started')
+        self.failIfFlagNotSet('triggeree_started')
+        self.failIfFlagSet('triggeree_finished')
+        self.failIfFlagSet('triggerer_finished')
+
 # TODO: test everything, from Change submission to Scheduler to Build to
 # Status. Use all the status types. Specifically I want to catch recurrences
 # of the bug where I forgot to make Waterfall inherit from StatusReceiver

docs/buildbot.texinfo

@@ -2186 +2186 @@
 scheduler is triggered. The next section (@pxref{Build Dependencies})
 describes this scheduler in more detail.
 
+@item Triggerable
+This scheduler does nothing until it is triggered by a Trigger
+step in another build.  This facilitates a more general form of
+build dependencies, as described in the next section (@pxref{Build
+Dependencies}).
+
 @item Periodic
 This simple scheduler just triggers a build every N seconds.
 
@@ -2212 +2218 @@
 @cindex Dependent
 @cindex Dependencies
 @slindex buildbot.scheduler.Dependent
+@slindex buildbot.scheduler.Triggerable
 
 It is common to wind up with one kind of build which should only be
 performed if the same source code was successfully handled by some
@@ -2254 +2261 @@
 @code{Scheduler} @emph{instance}, not a name. This makes it impossible
 to create circular dependencies in the config file.
 
+A more general way to coordinate builds is by ``triggering'' schedulers
+from builds.  The Triggerable waits to be triggered by a
+Trigger step in another build.  That step can optionally
+wait for the scheduler's builds to complete.  This provides two
+advantages over Dependent schedulers.  First, the same scheduler
+can be triggered from multiple builds.  Second, the ability to wait
+for a Triggerable's builds to complete provides a form of
+"subroutine call", where one or more builds can "call" a scheduler
+to perform some work for them,  perhaps on other buildslaves.
+
+@example
+from buildbot import scheduler
+from buildbot.steps import trigger
+checkin = scheduler.Scheduler("checkin", None, 5*60,
+                            ["checkin"])
+nightly = scheduler.Scheduler("nightly", ...
+                            ["nightly"])
+mktarball = scheduler.Triggerable("mktarball",
+                              ["mktarball"])
+build = scheduler.Triggerable("build-all-platforms",
+                              ["build-all-platforms"])
+test = scheduler.Triggerable("distributed-test",
+                              ["distributed-test"])
+package = scheduler.Triggerable("package-all-platforms",
+                              ["package-all-platforms"])
+c['schedulers'] = [checkin, nightly, build, test, package]
+
+checkin_factory = factory.BuildFactory()
+f.addStep(trigger.TriggerStep('mktarball',
+    schedulers=['mktarball'],
+    waitForFinish=1)
+f.addStep(trigger.TriggerStep('build',
+    schedulers=['build-all-platforms'],
+    waitForFinish=1)
+f.addStep(trigger.TriggerStep('test',
+    schedulers=['distributed-test'],
+    waitForFinish=1)
+
+nightly_factory = factory.BuildFactory()
+f.addStep(trigger.TriggerStep('mktarball',
+    schedulers=['mktarball'],
+    waitForFinish=1)
+f.addStep(trigger.TriggerStep('build',
+    schedulers=['build-all-platforms'],
+    waitForFinish=1)
+f.addStep(trigger.TriggerStep('package',
+    schedulers=['package-all-platforms'],
+    waitForFinish=1)
+@end example
 
 @node Setting the slaveport, Buildslave Specifiers, Listing Change Sources and Schedulers, Configuration
 @section Setting the slaveport
@@ -3660 +3716 @@
 * Simple ShellCommand Subclasses::  
 * Python BuildSteps::           
 * Transferring Files::          
+* Triggering Schedulers::
 * Writing New BuildSteps::      
 @end menu
 
@@ -4596 +4653 @@
 
 @end table
 
-@node Python BuildSteps, Transferring Files, Simple ShellCommand Subclasses, Build Steps
+@node Python BuildSteps, Triggering Schedulers, Simple ShellCommand Subclasses, Build Steps
 @subsection Python BuildSteps
 
 Here are some BuildSteps that are specifcally useful for projects
@@ -4672 +4729 @@
 @end example
 
 
-@node Transferring Files, Writing New BuildSteps, Python BuildSteps, Build Steps
+@node Transferring Files, Triggering Schedulers, Python BuildSteps, Build Steps
 @subsection Transferring Files
 
 @cindex File Transfer
@@ -4761 +4818 @@
 creation time (@pxref{Buildslave Options}).
 
 
-@node Writing New BuildSteps,  , Transferring Files, Build Steps
+@node Triggering Schedulers, Writing New BuildSteps, Transferring Files, Build Steps
+@subsection Triggering Schedulers
+
+The counterpart to the Triggerable described in section
+@pxref{Build Dependencies} is the Trigger BuildStep.
+
+@example
+from buildbot.steps.trigger import Trigger
+f.addStep(Trigger,
+          schedulers=['build-prep'],
+          waitForFinish=1,
+          updateSourceStamp=1)
+@end example
+
+The @code{schedulers=} argument lists the Triggerables
+that should be triggered when this step is executed.  Note that
+it is possible, but not advisable, to create a cycle where a build
+continually triggers itself, because the schedulers are specified
+by name.
+
+If @code{waitForFinish} is true, then the step will not finish until
+all of the builds from the triggered schedulers have finished.  If this
+argument is not given, then the buildstep succeeds immediately after
+triggering the schedulers.
+
+If @code{updateSourceStamp} is true, then step updates the SourceStamp
+given to the Triggerables to include @code{got_revision}
+(the revision actually used in this build) as @code{revision} (the
+revision to use in the triggered builds).  This is useful to ensure
+that all of the builds use exactly the same SourceStamp, even if
+other Changes have occurred while the build was running.
+
+@node Writing New BuildSteps,  , Triggering Schedulers, Build Steps
 @subsection Writing New BuildSteps
 
 While it is a good idea to keep your build process self-contained in