Ticket #56: trigger.patch

File trigger.patch, 14.1 KB (added by dustin, 13 years ago)

buildsteps can trigger schedulers

  • buildbot/scheduler.py

    A pair of a Scheduler and a BuildStep, where the step can "trigger"
    one or more schedulers, and optionally waits on their completion.
    
    This allows a general form of subroutine call, but is more specifically
    useful as a more powerful way to express dependent builds, particularly
    when a buid must take place cooperatively among a group of builders.
    old new  
    682682        from buildbot.status.client import makeRemote
    683683        return makeRemote(bs.status)
    684684
     685class Triggerable(BaseUpstreamScheduler):
     686    """
     687    This scheduler doesn't do anything until it is triggered by
     688    a Trigger step in a factory.
     689    """
     690
     691    def __init__(self, name, builderNames):
     692        BaseUpstreamScheduler.__init__(self, name)
     693        self.builderNames = builderNames
     694
     695    def listBuilderNames(self):
     696        return self.builderNames
     697
     698    def getPendingBuildTimes(self):
     699        return []
     700
     701    def trigger(self, ss):
     702        """
     703        Trigger this scheduler.  Returns a deferred that will fire when the buildset
     704        is finished.
     705        """
     706        bs = buildset.BuildSet(self.builderNames, ss)
     707        d = bs.waitUntilFinished()
     708        self.submit(bs)
     709        return d
  • new file triggers/buildbot/steps/trigger.py

    - +  
     1from buildbot.process.buildstep import LoggingBuildStep, SUCCESS, FAILURE, EXCEPTION
     2from buildbot.steps.shell import WithProperties
     3from buildbot.sourcestamp import SourceStamp
     4from buildbot.scheduler import Triggerable
     5from twisted.internet import defer
     6from twisted.python import log
     7import os
     8
     9class Trigger(LoggingBuildStep):
     10    """
     11    I trigger a Triggerable.  It's fun.
     12    """
     13    name = "trigger"
     14
     15    flunkOnFailure = True
     16
     17    def __init__(self,
     18        schedulers=[],
     19        updateSourceStamp=False,
     20        waitForFinish=False,
     21        **kwargs):
     22        """
     23        Trigger the given schedulers when this step is executed.
     24
     25        @var schedulers: list of schedulers' names that should be triggered.  Schedulers
     26        can be specified using WithProperties, if desired.
     27
     28        @var updateSourceStamp: should I update the source stamp to contain got_revision
     29        from the triggering build?
     30
     31        @var waitForFinish: should I wait for all of the triggered schedulers to finish
     32        their builds?
     33        """
     34        assert schedulers, "You must specify a scheduler to trigger"
     35        self.schedulers = schedulers
     36        self.updateSourceStamp = updateSourceStamp
     37        self.waitForFinish = waitForFinish
     38        self.running = False
     39        LoggingBuildStep.__init__(self, **kwargs)
     40
     41    def interrupt(self, reason):
     42        if self.running:
     43            self.step_status.setColor("red")
     44            self.step_status.setText(["interrupted"])
     45
     46    def start(self):
     47        self.running = True
     48        ss = self.build.getSourceStamp()
     49        if self.updateSourceStamp:
     50            ss = SourceStamp(ss.branch, self.build.getProperty('got_revision'), ss.patch)
     51        # (is there an easier way to find the BuildMaster?)
     52        all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
     53        all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
     54        unknown_schedulers = []
     55        triggered_schedulers = []
     56
     57        dl = []
     58        for scheduler in self.schedulers:
     59            if isinstance(scheduler, WithProperties):
     60                scheduler = scheduler.render(self.build)
     61            if all_schedulers.has_key(scheduler):
     62                sch = all_schedulers[scheduler]
     63                if isinstance(sch, Triggerable):
     64                    dl.append(sch.trigger(ss))
     65                    triggered_schedulers.append(scheduler)
     66                else:
     67                    unknown_schedulers.append(scheduler)
     68            else:
     69                unknown_schedulers.append(scheduler)
     70
     71        if unknown_schedulers:
     72            self.step_status.setColor("red")
     73            self.step_status.setText(['no scheduler:'] + unknown_schedulers)
     74            rc = FAILURE
     75        else:
     76            rc = SUCCESS
     77            self.step_status.setText(['triggered'] + triggered_schedulers)
     78            if self.waitForFinish:
     79                self.step_status.setColor("yellow")
     80            else:
     81                self.step_status.setColor("green")
     82
     83        if self.waitForFinish:
     84            d = defer.DeferredList(dl, consumeErrors=1)
     85        else:
     86            d = defer.succeed([])
     87
     88        def cb(rclist):
     89            rc = SUCCESS
     90            for was_cb, buildsetstatus in rclist:
     91                # TODO: make this algo more configurable
     92                if not was_cb:
     93                    rc = EXCEPTION
     94                    break
     95                if buildsetstatus.getResults() == FAILURE:
     96                    rc = FAILURE
     97            return self.finished(rc)
     98
     99        def eb(why):
     100            return self.finished(FAILURE)
     101
     102        d.addCallbacks(cb, eb)
  • buildbot/test/test_run.py

    old new  
    543543    def _testTestFlag_2(self, res):
    544544        self.failUnlessEqual(self.getFlag('foo'), 'bar')
    545545
     546class Triggers(RunMixin, TestFlagMixin, unittest.TestCase):
     547    config_trigger = config_base + """
     548from buildbot.scheduler import Triggerable, Scheduler
     549from buildbot.steps.trigger import Trigger
     550from buildbot.steps.dummy import Dummy
     551from buildbot.test.runutils import SetTestFlagStep
     552c['schedulers'] = [
     553    Scheduler('triggerer', None, 0.1, ['triggerer']),
     554    Triggerable('triggeree', ['triggeree'])
     555]
     556triggerer = factory.BuildFactory([
     557    s(SetTestFlagStep, flagname='triggerer_started'),
     558    s(Trigger, flunkOnFailure=True, @ARGS@),
     559    s(SetTestFlagStep, flagname='triggerer_finished'),
     560    ])
     561triggeree = factory.BuildFactory([
     562    s(SetTestFlagStep, flagname='triggeree_started'),
     563    s(@DUMMYCLASS@),
     564    s(SetTestFlagStep, flagname='triggeree_finished'),
     565    ])
     566c['builders'] = [{'name': 'triggerer', 'slavename': 'bot1',
     567                  'builddir': 'triggerer', 'factory': triggerer},
     568                 {'name': 'triggeree', 'slavename': 'bot1',
     569                  'builddir': 'triggeree', 'factory': triggeree}]
     570"""
     571
     572    def mkConfig(self, args, dummyclass="Dummy"):
     573        return self.config_trigger.replace("@ARGS@", args).replace("@DUMMYCLASS@", dummyclass)
     574
     575    def setupTest(self, args, dummyclass, checkFn):
     576        self.clearFlags()
     577        m = self.master
     578        m.loadConfig(self.mkConfig(args, dummyclass))
     579        m.readConfig = True
     580        m.startService()
     581
     582        c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
     583        m.change_svc.addChange(c)
     584
     585        d = self.connectSlave(builders=['triggerer', 'triggeree'])
     586        d.addCallback(self.startTimer, 0.5, checkFn)
     587        return d
     588
     589    def startTimer(self, res, time, next_fn):
     590        d = defer.Deferred()
     591        reactor.callLater(time, d.callback, None)
     592        d.addCallback(next_fn)
     593        return d
     594
     595    def testTriggerBuild(self):
     596        return self.setupTest("schedulers=['triggeree']",
     597                "Dummy",
     598                self._checkTriggerBuild)
     599
     600    def _checkTriggerBuild(self, res):
     601        self.failIfFlagNotSet('triggerer_started')
     602        self.failIfFlagNotSet('triggeree_started')
     603        self.failIfFlagSet('triggeree_finished')
     604        self.failIfFlagNotSet('triggerer_finished')
     605
     606    def testTriggerBuildWait(self):
     607        return self.setupTest("schedulers=['triggeree'], waitForFinish=1",
     608                "Dummy",
     609                self._checkTriggerBuildWait)
     610
     611    def _checkTriggerBuildWait(self, res):
     612        self.failIfFlagNotSet('triggerer_started')
     613        self.failIfFlagNotSet('triggeree_started')
     614        self.failIfFlagSet('triggeree_finished')
     615        self.failIfFlagSet('triggerer_finished')
     616
    546617# TODO: test everything, from Change submission to Scheduler to Build to
    547618# Status. Use all the status types. Specifically I want to catch recurrences
    548619# of the bug where I forgot to make Waterfall inherit from StatusReceiver
  • docs/buildbot.texinfo

    old new  
    21022102section (@pxref{Build Dependencies}) describes this scheduler in more
    21032103detail.
    21042104
     2105@item Triggerable
     2106This scheduler does nothing until it is triggered by a Trigger
     2107step in another build.  This facilitates a more general form of
     2108build dependencies, as described in the next section (@pxref{Build
     2109Dependencies}).
     2110
    21052111@item Periodic
    21062112This simple scheduler just triggers a build every N seconds.
    21072113
     
    21282134@cindex Dependent
    21292135@cindex Dependencies
    21302136@slindex buildbot.scheduler.Dependent
     2137@slindex buildbot.scheduler.Triggerable
    21312138
    21322139It is common to wind up with one kind of build which should only be
    21332140performed if the same source code was successfully handled by some
     
    21702177@code{Scheduler} @emph{instance}, not a name. This makes it impossible
    21712178to create circular dependencies in the config file.
    21722179
     2180A more general way to coordinate builds is by ``triggering'' schedulers
     2181from builds.  The Triggerable waits to be triggered by a
     2182Trigger step in another build.  That step can optionally
     2183wait for the scheduler's builds to complete.  This provides two
     2184advantages over Dependent schedulers.  First, the same scheduler
     2185can be triggered from multiple builds.  Second, the ability to wait
     2186for a Triggerable's builds to complete provides a form of
     2187"subroutine call", where one or more builds can "call" a scheduler
     2188to perform some work for them,  perhaps on other buildslaves.
     2189
     2190@example
     2191from buildbot import scheduler
     2192from buildbot.steps import trigger
     2193checkin = scheduler.Scheduler("checkin", None, 5*60,
     2194                            ["checkin"])
     2195nightly = scheduler.Scheduler("nightly", ...
     2196                            ["nightly"])
     2197mktarball = scheduler.Triggerable("mktarball",
     2198                              ["mktarball"])
     2199build = scheduler.Triggerable("build-all-platforms",
     2200                              ["build-all-platforms"])
     2201test = scheduler.Triggerable("distributed-test",
     2202                              ["distributed-test"])
     2203package = scheduler.Triggerable("package-all-platforms",
     2204                              ["package-all-platforms"])
     2205c['schedulers'] = [checkin, nightly, build, test, package]
     2206
     2207checkin_factory = factory.BuildFactory()
     2208f.addStep(trigger.TriggerStep('mktarball',
     2209    schedulers=['mktarball'],
     2210    waitForFinish=1)
     2211f.addStep(trigger.TriggerStep('build',
     2212    schedulers=['build-all-platforms'],
     2213    waitForFinish=1)
     2214f.addStep(trigger.TriggerStep('test',
     2215    schedulers=['distributed-test'],
     2216    waitForFinish=1)
     2217
     2218nightly_factory = factory.BuildFactory()
     2219f.addStep(trigger.TriggerStep('mktarball',
     2220    schedulers=['mktarball'],
     2221    waitForFinish=1)
     2222f.addStep(trigger.TriggerStep('build',
     2223    schedulers=['build-all-platforms'],
     2224    waitForFinish=1)
     2225f.addStep(trigger.TriggerStep('package',
     2226    schedulers=['package-all-platforms'],
     2227    waitForFinish=1)
     2228@end example
    21732229
    21742230@node Setting the slaveport, Buildslave Specifiers, Listing Change Sources and Schedulers, Configuration
    21752231@section Setting the slaveport
     
    32113267* Simple ShellCommand Subclasses:: 
    32123268* Python BuildSteps::           
    32133269* Transferring Files::         
     3270* Triggering Schedulers::
    32143271* Writing New BuildSteps::     
    32153272@end menu
    32163273
     
    40824139
    40834140@end table
    40844141
    4085 @node Python BuildSteps, Transferring Files, Simple ShellCommand Subclasses, Build Steps
     4142@node Python BuildSteps, Triggering Schedulers, Simple ShellCommand Subclasses, Build Steps
    40864143@subsection Python BuildSteps
    40874144
    40884145Here are some BuildSteps that are specifcally useful for projects
     
    41584215@end example
    41594216
    41604217
    4161 @node Transferring Files, Writing New BuildSteps, Python BuildSteps, Build Steps
     4218@node Transferring Files, Triggering Schedulers, Python BuildSteps, Build Steps
    41624219@subsection Transferring Files
    41634220
    41644221@cindex File Transfer
     
    42474304creation time (@pxref{Buildslave Options}).
    42484305
    42494306
    4250 @node Writing New BuildSteps,  , Transferring Files, Build Steps
     4307@node Triggering Schedulers, Writing New BuildSteps, Transferring Files, Build Steps
     4308@subsection Triggering Schedulers
     4309
     4310The counterpart to the Triggerable described in section
     4311@pxref{Build Dependencies} is the Trigger BuildStep.
     4312
     4313@example
     4314from buildbot.steps.trigger import Trigger
     4315f.addStep(Trigger,
     4316          schedulers=['build-prep'],
     4317          waitForFinish=1,
     4318          updateSourceStamp=1)
     4319@end example
     4320
     4321The @code{schedulers=} argument lists the Triggerables
     4322that should be triggered when this step is executed.  Note that
     4323it is possible, but not advisable, to create a cycle where a build
     4324continually triggers itself, because the schedulers are specified
     4325by name.
     4326
     4327If @code{waitForFinish} is true, then the step will not finish until
     4328all of the builds from the triggered schedulers have finished.  If this
     4329argument is not given, then the buildstep succeeds immediately after
     4330triggering the schedulers.
     4331
     4332If @code{updateSourceStamp} is true, then step updates the SourceStamp
     4333given to the Triggerables to include @code{got_revision}
     4334(the revision actually used in this build) as @code{revision} (the
     4335revision to use in the triggered builds).  This is useful to ensure
     4336that all of the builds use exactly the same SourceStamp, even if
     4337other Changes have occurred while the build was running.
     4338
     4339@node Writing New BuildSteps,  , Triggering Schedulers, Build Steps
    42514340@subsection Writing New BuildSteps
    42524341
    42534342While it is a good idea to keep your build process self-contained in