[PATCH] Make autobuild on newer ubuntu possible

Tue Apr 14 15:46:39 MDT 2015

Reviewed-By: me

Cheers,

Jelmer

On Tue, Apr 14, 2015 at 09:17:41PM +1200, Andrew Bartlett wrote:
> On Tue, 2015-04-14 at 12:27 +1200, Andrew Bartlett wrote:
> > On Mon, 2015-04-13 at 12:28 +0200, Jelmer Vernooij wrote:
> > 
> > > Could we rather just blacklist those files, instead of allowing a blanket
> > > "ignore all changes" ?
> > > 
> > > This problem seems intrinsic to all files that are autogenerated *and* checked
> > > in if there are multiple versions of the generator out there. We don't
> > > have many files that fall into this category - the pidl files are the only I
> > > can think of at the moment.
> > 
> > I think you will prefer this patch set then.
> > 
> > I also fix up another pet annoyance of mine, that is when testing
> > changes like this, we now automatically skip the 60-600 second wait for
> > modules other than the main samba build.
> > 
> > Please review/push.
> 
> The attached patch helps us better use temporary cloud hosts to run an
> autobuild.  Combined with scripts to run a autobuild in a fresh cloud
> host, this 
> 
> My aim here is to, using as much of autobuild as possible, make it
> trivial to push 'try' branches at a cloud (such as the Catalyst Cloud),
> and get a result returned by e-mail.  In particular I'm keen to do this
> without adding load to sn-devel, and to be able to do parallel runs
> without the limitations of that hardware.
> 
> Andrew Bartlett
> 
> -- 
> Andrew Bartlett                       http://samba.org/~abartlet/
> Authentication Developer, Samba Team  http://samba.org
> Samba Developer, Catalyst IT          http://catalyst.net.nz/services/samba
> 

> From c94d2698414245a79d416280cf348f23ff390730 Mon Sep 17 00:00:00 2001
> From: Andrew Bartlett <abartlet at samba.org>
> Date: Tue, 14 Apr 2015 17:18:02 +1200
> Subject: [PATCH] autobuild: Add options to set mail host and send e-mail with
>  logs
> 
> This helps when running the script in a cloud instance as
> it is cheapest to shut it down once run
> 
> Signed-off-by: Andrew Bartlett <abartlet at samba.org>
> ---
>  script/autobuild.py | 68 +++++++++++++++++++++++++++++++++++------------------
>  1 file changed, 45 insertions(+), 23 deletions(-)
> 
> diff --git a/script/autobuild.py b/script/autobuild.py
> index 3c455bd..b2303f7 100755
> --- a/script/autobuild.py
> +++ b/script/autobuild.py
> @@ -7,8 +7,13 @@ from subprocess import call, check_call,Popen, PIPE
>  import os, tarfile, sys, time
>  from optparse import OptionParser
>  import smtplib
> +import email
>  from email.mime.text import MIMEText
> +from email.mime.base import MIMEBase
> +from email.mime.application import MIMEApplication
> +from email.mime.multipart import MIMEMultipart
>  from distutils.sysconfig import get_python_lib
> +import platform
>  
>  # This speeds up testing remarkably.
>  os.environ['TDB_NO_FSYNC'] = '1'
> @@ -461,6 +466,10 @@ parser.add_option("", "--retry", help="automatically retry if master changes",
>                    default=False, action="store_true")
>  parser.add_option("", "--email", help="send email to the given address on failure",
>                    type='str', default=None)
> +parser.add_option("", "--email-from", help="send email from the given address",
> +                  type='str', default="autobuild at samba.org")
> +parser.add_option("", "--email-server", help="send email via the given server",
> +                  type='str', default='localhost')
>  parser.add_option("", "--always-email", help="always send email, even on success",
>                    action="store_true")
>  parser.add_option("", "--daemon", help="daemonize after initial setup",
> @@ -469,6 +478,29 @@ parser.add_option("", "--branch", help="the branch to work on (default=master)",
>                    default="master", type='str')
>  parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
>                    default=gitroot, type='str')
> +parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
> +                  default=False, action="store_true")
> +
> +def send_email(subject, text, log_tar):
> +    outer = MIMEMultipart()
> +    outer['Subject'] = subject
> +    outer['To'] = options.email
> +    outer['From'] = options.email_from
> +    outer['Date'] = email.utils.formatdate(localtime = True)
> +    outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
> +    outer.attach(MIMEText(text, 'plain'))
> +    if options.attach_logs:
> +        fp = open(log_tar, 'rb')
> +        msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
> +        fp.close()
> +        # Set the filename parameter
> +        msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
> +        outer.attach(msg)
> +    content = outer.as_string()
> +    s = smtplib.SMTP(options.email_server)
> +    s.sendmail(options.email_from, [options.email], content)
> +    s.set_debuglevel(1)
> +    s.quit()
>  
>  def email_failure(status, failed_task, failed_stage, failed_tag, errstr, log_base=None):
>      '''send an email to options.email about the failure'''
> @@ -478,7 +510,7 @@ def email_failure(status, failed_task, failed_stage, failed_tag, errstr, log_bas
>      text = '''
>  Dear Developer,
>  
> -Your autobuild failed when trying to test %s with the following error:
> +Your autobuild on %s failed when trying to test %s with the following error:
>     %s
>  
>  the autobuild has been abandoned. Please fix the error and resubmit.
> @@ -486,7 +518,7 @@ the autobuild has been abandoned. Please fix the error and resubmit.
>  A summary of the autobuild process is here:
>  
>    %s/autobuild.log
> -''' % (failed_task, errstr, log_base)
> +''' % (platform.node(), failed_task, errstr, log_base)
>      
>      if failed_task != 'rebase':
>          text += '''
> @@ -504,15 +536,11 @@ The top commit for the tree that was built was:
>  %s
>  
>  ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
> -    msg = MIMEText(text)
> -    msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage)
> -    msg['From'] = 'autobuild at samba.org'
> -    msg['To'] = options.email
> -
> -    s = smtplib.SMTP()
> -    s.connect()
> -    s.sendmail(msg['From'], [msg['To']], msg.as_string())
> -    s.quit()
> +
> +    logs = os.path.join(gitroot, 'logs.tar.gz')
> +    send_email('autobuild failure on %s for task %s during %s'
> +               % (platform.node(), failed_task, failed_stage),
> +               text, logs)
>  
>  def email_success(log_base=None):
>      '''send an email to options.email about a successful build'''
> @@ -522,9 +550,9 @@ def email_success(log_base=None):
>      text = '''
>  Dear Developer,
>  
> -Your autobuild has succeeded.
> +Your autobuild on %s has succeeded.
>  
> -'''
> +''' % platform.node()
>  
>      if options.keeplogs:
>          text += '''
> @@ -541,15 +569,9 @@ The top commit for the tree that was built was:
>  %s
>  ''' % top_commit_msg
>  
> -    msg = MIMEText(text)
> -    msg['Subject'] = 'autobuild success'
> -    msg['From'] = 'autobuild at samba.org'
> -    msg['To'] = options.email
> -
> -    s = smtplib.SMTP()
> -    s.connect()
> -    s.sendmail(msg['From'], [msg['To']], msg.as_string())
> -    s.quit()
> +    logs = os.path.join(gitroot, 'logs.tar.gz')
> +    send_email('autobuild sucess on %s ' % platform.node(),
> +               text, logs)
>  
>  
>  (options, args) = parser.parse_args()
> @@ -622,7 +644,7 @@ if status == 0:
>          run_cmd(options.passcmd, dir=test_master)
>      if options.pushto is not None:
>          push_to(options.pushto, push_branch=options.branch)
> -    if options.keeplogs:
> +    if options.keeplogs or options.attach_logs:
>          blist.tarlogs("logs.tar.gz")
>          print("Logs in logs.tar.gz")
>      if options.always_email:
> -- 
> 2.1.4
>