Ticket #74: 081p1_mail.py

File 081p1_mail.py, 22.2 KB (added by umesh4, 19 months ago)
Line 
1# -*- test-case-name: buildbot.test.test_status -*-
2
3import re
4
5from email.Message import Message
6from email.Utils import formatdate
7from email.MIMEText import MIMEText
8from email.MIMEMultipart import MIMEMultipart
9from StringIO import StringIO
10import urllib
11
12from zope.interface import implements
13from twisted.internet import defer, reactor
14from twisted.mail.smtp import sendmail, ESMTPSenderFactory
15from twisted.python import log as twlog
16
17try:
18    from twisted.internet import ssl
19    from OpenSSL.SSL import SSLv3_METHOD
20except ImportError:
21    pass
22
23from buildbot import interfaces, util
24from buildbot.status import base
25from buildbot.status.builder import FAILURE, SUCCESS, WARNINGS, Results
26
27VALID_EMAIL = re.compile("[a-zA-Z0-9\.\_\%\-\+]+@[a-zA-Z0-9\.\_\%\-]+.[a-zA-Z]{2,6}")
28
29ENCODING = 'utf8'
30
31class Domain(util.ComparableMixin):
32    implements(interfaces.IEmailLookup)
33    compare_attrs = ["domain"]
34
35    def __init__(self, domain):
36        assert "@" not in domain
37        self.domain = domain
38
39    def getAddress(self, name):
40        """If name is already an email address, pass it through."""
41        if '@' in name:
42            return name
43        return name + "@" + self.domain
44
45
46def defaultMessage(mode, name, build, results, master_status):
47    """Generate a buildbot mail message and return a tuple of message text
48        and type."""
49    result = Results[results]
50    ss = build.getSourceStamp()
51
52    text = ""
53    if mode == "all":
54        text += "The Buildbot has finished a build"
55    elif mode == "failing":
56        text += "The Buildbot has detected a failed build"
57    elif attrs['mode'] == "warnings":
58        text += "The Buildbot has detected a problem in the build"
59    elif mode == "passing":
60        text += "The Buildbot has detected a passing build"
61    elif mode == "change" and result == 'success':
62        text += "The Buildbot has detected a restored build"
63    else:   
64        text += "The Buildbot has detected a new failure"
65    if ss and ss.project:
66        project = ss.project
67    else:
68        project = master_status.getProjectName()
69    text += " on builder %s while building %s.\n" % (name, project)
70    if master_status.getURLForThing(build):
71        text += "Full details are available at:\n %s\n" % master_status.getURLForThing(build)
72    text += "\n"
73
74    if master_status.getBuildbotURL():
75        text += "Buildbot URL: %s\n\n" % urllib.quote(master_status.getBuildbotURL(), '/:')
76
77    text += "Buildslave for this Build: %s\n\n" % build.getSlavename()
78    text += "Build Reason: %s\n" % build.getReason()
79
80    source = ""
81    if ss and ss.branch:
82        source += "[branch %s] " % ss.branch
83    if ss and ss.revision:
84        source += str(ss.revision)
85    else:
86        source += "HEAD"
87    if ss and ss.patch:
88        source += " (plus patch)"
89
90    text += "Build Source Stamp: %s\n" % source
91
92    text += "Blamelist: %s\n" % ",".join(build.getResponsibleUsers())
93
94    text += "\n"
95
96    t = build.getText()
97    if t:
98        t = ": " + " ".join(t)
99    else:
100        t = ""
101
102    if result == 'success':
103        text += "Build succeeded!\n"
104    elif result == 'warnings':
105        text += "Build Had Warnings%s\n" % t
106    else:
107        text += "BUILD FAILED%s\n" % t
108
109    text += "\n"
110    text += "sincerely,\n"
111    text += " -The Buildbot\n"
112    text += "\n"
113    return { 'body' : text, 'type' : 'plain' }
114
115class MailNotifier(base.StatusReceiverMultiService):
116    """This is a status notifier which sends email to a list of recipients
117    upon the completion of each build. It can be configured to only send out
118    mail for certain builds, and only send messages when the build fails, or
119    when it transitions from success to failure. It can also be configured to
120    include various build logs in each message.
121
122    By default, the message will be sent to the Interested Users list, which
123    includes all developers who made changes in the build. You can add
124    additional recipients with the extraRecipients argument.
125
126    To get a simple one-message-per-build (say, for a mailing list), use
127    sendToInterestedUsers=False, extraRecipients=['listaddr@example.org']
128
129    Each MailNotifier sends mail to a single set of recipients. To send
130    different kinds of mail to different recipients, use multiple
131    MailNotifiers.
132    """
133
134    implements(interfaces.IEmailSender)
135
136    compare_attrs = ["extraRecipients", "lookup", "fromaddr", "mode",
137                     "categories", "builders", "addLogs", "relayhost",
138                     "subject", "sendToInterestedUsers", "customMesg",
139                     "messageFormatter", "extraHeaders"]
140
141    def __init__(self, fromaddr, mode="all", categories=None, builders=None,
142                 addLogs=False, relayhost="localhost",
143                 subject="buildbot %(result)s in %(projectName)s on %(builder)s",
144                 lookup=None, extraRecipients=[],
145                 sendToInterestedUsers=True, customMesg=None,
146                 messageFormatter=defaultMessage, extraHeaders=None,
147                 addPatch=True, useTls=False, 
148                 smtpUser=None, smtpPassword=None, smtpPort=25):
149        """
150        @type  fromaddr: string
151        @param fromaddr: the email address to be used in the 'From' header.
152        @type  sendToInterestedUsers: boolean
153        @param sendToInterestedUsers: if True (the default), send mail to all
154                                      of the Interested Users. If False, only
155                                      send mail to the extraRecipients list.
156
157        @type  extraRecipients: tuple of string
158        @param extraRecipients: a list of email addresses to which messages
159                                should be sent (in addition to the
160                                InterestedUsers list, which includes any
161                                developers who made Changes that went into this
162                                build). It is a good idea to create a small
163                                mailing list and deliver to that, then let
164                                subscribers come and go as they please.
165
166        @type  subject: string
167        @param subject: a string to be used as the subject line of the message.
168                        %(builder)s will be replaced with the name of the
169                        builder which provoked the message.
170
171        @type  mode: string (defaults to all)
172        @param mode: one of:
173                     - 'all': send mail about all builds, passing and failing
174                     - 'failing': only send mail about builds which fail
175                     - 'warnings': send mail if builds contain warnings or fail
176                     - 'passing': only send mail about builds which succeed
177                     - 'problem': only send mail about a build which failed
178                     when the previous build passed
179                     - 'change': only send mail about builds who change status
180
181        @type  builders: list of strings
182        @param builders: a list of builder names for which mail should be
183                         sent. Defaults to None (send mail for all builds).
184                         Use either builders or categories, but not both.
185
186        @type  categories: list of strings
187        @param categories: a list of category names to serve status
188                           information for. Defaults to None (all
189                           categories). Use either builders or categories,
190                           but not both.
191
192        @type  addLogs: boolean
193        @param addLogs: if True, include all build logs as attachments to the
194                        messages.  These can be quite large. This can also be
195                        set to a list of log names, to send a subset of the
196                        logs. Defaults to False.
197
198        @type  addPatch: boolean
199        @param addPatch: if True, include the patch when the source stamp
200                         includes one.
201
202        @type  relayhost: string
203        @param relayhost: the host to which the outbound SMTP connection
204                          should be made. Defaults to 'localhost'
205
206        @type  lookup:    implementor of {IEmailLookup}
207        @param lookup:    object which provides IEmailLookup, which is
208                          responsible for mapping User names (which come from
209                          the VC system) into valid email addresses. If not
210                          provided, the notifier will only be able to send mail
211                          to the addresses in the extraRecipients list. Most of
212                          the time you can use a simple Domain instance. As a
213                          shortcut, you can pass as string: this will be
214                          treated as if you had provided Domain(str). For
215                          example, lookup='twistedmatrix.com' will allow mail
216                          to be sent to all developers whose SVN usernames
217                          match their twistedmatrix.com account names.
218                         
219        @type  customMesg: func
220        @param customMesg: (this function is deprecated)
221
222        @type  messageFormatter: func
223        @param messageFormatter: function taking (mode, name, build, result,
224                                 master_status ) and returning a dictionary
225                                 containing two required keys "body" and "type",
226                                 with a third optional key, "subject". The
227                                 "body" key gives a string that contains the
228                                 complete text of the message. The "type" key
229                                 is the message type ('plain' or 'html'). The
230                                 'html' type should be used when generating an
231                                 HTML message.  The optional "subject" key
232                                 gives the subject for the email.
233
234        @type  extraHeaders: dict
235        @param extraHeaders: A dict of extra headers to add to the mail. It's
236                             best to avoid putting 'To', 'From', 'Date',
237                             'Subject', or 'CC' in here. Both the names and
238                             values may be WithProperties instances.
239
240        @type useTls: boolean
241        @param useTls: Send emails using TLS and authenticate with the
242                       smtp host. Defaults to False.
243
244        @type smtpUser: string
245        @param smtpUser: The user that will attempt to authenticate with the
246                         relayhost when useTls is True.
247
248        @type smtpPassword: string
249        @param smtpPassword: The password that smtpUser will use when
250                             authenticating with relayhost.
251
252        @type smtpPort: int
253        @param smtpPort: The port that will be used when connecting to the
254                         relayhost. Defaults to 25.
255        """
256
257        base.StatusReceiverMultiService.__init__(self)
258        assert isinstance(extraRecipients, (list, tuple))
259        for r in extraRecipients:
260            assert isinstance(r, str)
261            assert VALID_EMAIL.search(r) # require full email addresses, not User names
262        self.extraRecipients = extraRecipients
263        self.sendToInterestedUsers = sendToInterestedUsers
264        self.fromaddr = fromaddr
265        assert mode in ('all', 'failing', 'problem', 'change', 'passing', 'warnings')
266        self.mode = mode
267        self.categories = categories
268        self.builders = builders
269        self.addLogs = addLogs
270        self.relayhost = relayhost
271        self.subject = subject
272        if lookup is not None:
273            if type(lookup) is str:
274                lookup = Domain(lookup)
275            assert interfaces.IEmailLookup.providedBy(lookup)
276        self.lookup = lookup
277        self.customMesg = customMesg
278        self.messageFormatter = messageFormatter
279        if extraHeaders:
280            assert isinstance(extraHeaders, dict)
281        self.extraHeaders = extraHeaders
282        self.addPatch = addPatch
283        self.useTls = useTls
284        self.smtpUser = smtpUser
285        self.smtpPassword = smtpPassword
286        self.smtpPort = smtpPort
287        self.watched = []
288        self.master_status = None
289
290        # you should either limit on builders or categories, not both
291        if self.builders != None and self.categories != None:
292            twlog.err("Please specify only builders to ignore or categories to include")
293            raise # FIXME: the asserts above do not raise some Exception either
294
295        if customMesg:
296            twlog.msg("customMesg is deprecated; please use messageFormatter instead")
297
298    def setServiceParent(self, parent):
299        """
300        @type  parent: L{buildbot.master.BuildMaster}
301        """
302        base.StatusReceiverMultiService.setServiceParent(self, parent)
303        self.setup()
304
305    def setup(self):
306        self.master_status = self.parent.getStatus()
307        self.master_status.subscribe(self)
308
309    def disownServiceParent(self):
310        self.master_status.unsubscribe(self)
311        for w in self.watched:
312            w.unsubscribe(self)
313        return base.StatusReceiverMultiService.disownServiceParent(self)
314
315    def builderAdded(self, name, builder):
316        # only subscribe to builders we are interested in
317        if self.categories != None and builder.category not in self.categories:
318            return None
319
320        self.watched.append(builder)
321        return self # subscribe to this builder
322
323    def builderRemoved(self, name):
324        pass
325
326    def builderChangedState(self, name, state):
327        pass
328    def buildStarted(self, name, build):
329        pass
330    def buildFinished(self, name, build, results):
331        # here is where we actually do something.
332        builder = build.getBuilder()
333        if self.builders is not None and name not in self.builders:
334            return # ignore this build
335        if self.categories is not None and \
336               builder.category not in self.categories:
337            return # ignore this build
338
339        if self.mode == "warnings" and results == SUCCESS:
340            return
341        if self.mode == "failing" and results != FAILURE:
342            return
343        if self.mode == "passing" and results != SUCCESS:
344            return
345        if self.mode == "problem":
346            if results != FAILURE:
347                return
348            prev = build.getPreviousBuild()
349            if prev and prev.getResults() == FAILURE:
350                return
351        if self.mode == "change":
352            prev = build.getPreviousBuild()
353            if not prev or prev.getResults() == results:
354                return
355        # for testing purposes, buildMessage returns a Deferred that fires
356        # when the mail has been sent. To help unit tests, we return that
357        # Deferred here even though the normal IStatusReceiver.buildFinished
358        # signature doesn't do anything with it. If that changes (if
359        # .buildFinished's return value becomes significant), we need to
360        # rearrange this.
361        return self.buildMessage(name, build, results)
362
363    def getCustomMesgData(self, mode, name, build, results, master_status):
364        #
365        # logs is a list of tuples that contain the log
366        # name, log url, and the log contents as a list of strings.
367        #
368        logs = list()
369        for logf in build.getLogs():
370            logStep = logf.getStep()
371            stepName = logStep.getName()
372            logStatus, dummy = logStep.getResults()
373            logName = logf.getName()
374            logs.append(('%s.%s' % (stepName, logName),
375                         '%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), stepName, logName),
376                         logf.getText().splitlines(),
377                         logStatus))
378
379        attrs = {'builderName': name,
380                 'projectName': master_status.getProjectName(),
381                 'mode': mode,
382                 'result': Results[results],
383                 'buildURL': master_status.getURLForThing(build),
384                 'buildbotURL': master_status.getBuildbotURL(),
385                 'buildText': build.getText(),
386                 'buildProperties': build.getProperties(),
387                 'slavename': build.getSlavename(),
388                 'reason':  build.getReason(),
389                 'responsibleUsers': build.getResponsibleUsers(),
390                 'branch': "",
391                 'revision': "",
392                 'patch': "",
393                 'changes': [],
394                 'logs': logs}
395
396        ss = build.getSourceStamp()
397        if ss:
398            attrs['branch'] = ss.branch
399            attrs['revision'] = ss.revision
400            attrs['patch'] = ss.patch
401            attrs['changes'] = ss.changes[:]
402
403        return attrs
404
405    def createEmail(self, msgdict, builderName, projectName, results, build,
406                    patch=None, logs=None):
407        text = msgdict['body'].encode(ENCODING)
408        type = msgdict['type']
409        if 'subject' in msgdict:
410            subject = msgdict['subject'].encode(ENCODING)
411        else:
412            subject = self.subject % { 'result': Results[results],
413                                       'projectName': projectName,
414                                       'builder': builderName,
415                                       }
416
417
418        assert type in ('plain', 'html'), "'%s' message type must be 'plain' or 'html'." % type
419
420        if patch or logs:
421            m = MIMEMultipart()
422            m.attach(MIMEText(text, type, ENCODING))
423        else:
424            m = Message()
425            m.set_payload(text, ENCODING)
426            m.set_type("text/%s" % type)
427
428        m['Date'] = formatdate(localtime=True)
429        m['Subject'] = subject
430        m['From'] = self.fromaddr
431        # m['To'] is added later
432
433        if patch:
434            a = MIMEText(patch[1].encode(ENCODING), _charset=ENCODING)
435            a.add_header('Content-Disposition', "attachment",
436                         filename="source patch")
437            m.attach(a)
438        if logs:
439            for log in logs:
440                name = "%s.%s" % (log.getStep().getName(),
441                                  log.getName())
442                if self._shouldAttachLog(log.getName()) or self._shouldAttachLog(name):
443                    a = MIMEText(log.getText().encode(ENCODING), 
444                                 _charset=ENCODING)
445                    a.add_header('Content-Disposition', "attachment",
446                                 filename=name)
447                    m.attach(a)
448
449        # Add any extra headers that were requested, doing WithProperties
450        # interpolation if necessary
451        if self.extraHeaders:
452            properties = build.getProperties()
453            for k,v in self.extraHeaders.items():
454                k = properties.render(k)
455                if k in m:
456                    twlog("Warning: Got header " + k + " in self.extraHeaders "
457                          "but it already exists in the Message - "
458                          "not adding it.")
459                    continue
460                m[k] = properties.render(v)
461
462        return m
463
464    def buildMessage(self, name, build, results):
465        if self.customMesg:
466            # the customMesg stuff can be *huge*, so we prefer not to load it
467            attrs = self.getCustomMesgData(self.mode, name, build, results, self.master_status)
468            text, type = self.customMesg(attrs)
469            msgdict = { 'body' : text, 'type' : type }
470        else:
471            msgdict = self.messageFormatter(self.mode, name, build, results, self.master_status)
472
473        patch = None
474        ss = build.getSourceStamp()
475        if ss and ss.patch and self.addPatch:
476            patch == ss.patch
477        logs = None
478        if self.addLogs:
479            logs = build.getLogs()
480            twlog.err("LOG: %s" % str(logs))
481        m = self.createEmail(msgdict, name, self.master_status.getProjectName(),
482                             results, build, patch, logs)
483
484        # now, who is this message going to?
485        dl = []
486        recipients = []
487        if self.sendToInterestedUsers and self.lookup:
488            for u in build.getInterestedUsers():
489                d = defer.maybeDeferred(self.lookup.getAddress, u)
490                d.addCallback(recipients.append)
491                dl.append(d)
492        d = defer.DeferredList(dl)
493        d.addCallback(self._gotRecipients, recipients, m)
494        return d
495
496    def _shouldAttachLog(self, logname):
497        if type(self.addLogs) is bool:
498            return self.addLogs
499        return logname in self.addLogs
500
501    def _gotRecipients(self, res, rlist, m):
502        recipients = set()
503
504        for r in rlist:
505            if r is None: # getAddress didn't like this address
506                continue
507
508            # Git can give emails like 'User' <user@foo.com>@foo.com so check
509            # for two @ and chop the last
510            if r.count('@') > 1:
511                r = r[:r.rindex('@')]
512
513            if VALID_EMAIL.search(r):
514                recipients.add(r)
515            else:
516                twlog.msg("INVALID EMAIL: %r" + r)
517
518        # if we're sending to interested users move the extra's to the CC
519        # list so they can tell if they are also interested in the change
520        # unless there are no interested users
521        if self.sendToInterestedUsers and len(recipients):
522            extra_recips = self.extraRecipients[:]
523            extra_recips.sort()
524            m['CC'] = ", ".join(extra_recips)
525        else:
526            [recipients.add(r) for r in self.extraRecipients[:]]
527
528        rlist = list(recipients)
529        rlist.sort()
530        m['To'] = ", ".join(rlist)
531
532        # The extras weren't part of the TO list so add them now
533        if self.sendToInterestedUsers:
534            for r in self.extraRecipients:
535                recipients.add(r)
536
537        return self.sendMessage(m, list(recipients))
538
539    def tls_sendmail(self, s, recipients):
540        client_factory = ssl.ClientContextFactory()
541        client_factory.method = SSLv3_METHOD
542
543        result = defer.Deferred()
544
545       
546        sender_factory = ESMTPSenderFactory(
547            self.smtpUser, self.smtpPassword,
548            self.fromaddr, recipients, StringIO(s),
549            result, contextFactory=client_factory)
550
551        reactor.connectTCP(self.relayhost, self.smtpPort, sender_factory)
552       
553        return result
554
555    def sendMessage(self, m, recipients):
556        s = m.as_string()
557        twlog.msg("sending mail (%d bytes) to" % len(s), recipients)
558        if self.useTls:
559            return self.tls_sendmail(s, recipients)
560        else:
561            return sendmail(self.relayhost, self.fromaddr, recipients, s,
562                            port=self.smtpPort)