[SCM] build.samba.org - branch master updated

Jelmer Vernooij jelmer at samba.org
Wed Nov 10 03:23:13 MST 2010


The branch, master has been updated
       via  211abe8 More improvements to import-and-analyse.
      from  ee296a4 Simplify some code.

http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master


- Log -----------------------------------------------------------------
commit 211abe8c5db88a77017077495fd37f3a7d7cf019
Author: Jelmer Vernooij <jelmer at samba.org>
Date:   Wed Nov 10 11:23:57 2010 +0100

    More improvements to import-and-analyse.

-----------------------------------------------------------------------

Summary of changes:
 buildfarm/__init__.py |    4 +-
 buildfarm/data.py     |   96 ++++++++++++++-----------------------------------
 buildfarm/util.py     |    3 +-
 import-and-analyse.py |   18 ++++-----
 4 files changed, 38 insertions(+), 83 deletions(-)


Changeset truncated at 500 lines:

diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index b12fb7b..2370940 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -71,7 +71,7 @@ class BuildFarm(object):
             raise Exception("web directory %s does not exist" % self.webdir)
         self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
         self.builds = self._open_build_results()
-        self.upload_builds = self._open_upload_builds_results()
+        self.upload_builds = self._open_upload_build_results()
         self.hostdb = self._open_hostdb()
         self.compilers = self._load_compilers()
         self.lcovdir = os.path.join(self.path, "lcov/data")
@@ -131,7 +131,7 @@ class BuildFarm(object):
 
     def get_new_builds(self):
         from buildfarm import data
-        for host in self.hostsdb.hosts():
+        for host in self.hostdb.hosts():
             for tree in self.trees:
                 for compiler in self.compilers:
                     # By building the log file name this way, using only the list of
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 7062431..493badb 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -175,6 +175,10 @@ class Build(object):
         self.host = host
         self.compiler = compiler
         self.rev = rev
+        if rev is None:
+            self.basename = self._store.build_fname(self.tree, self.host, self.compiler)
+        else:
+            self.basename = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
 
     ###################
     # the mtime age is used to determine if builds are still happening
@@ -183,26 +187,22 @@ class Build(object):
 
     def age_mtime(self):
         """get the age of build from mtime"""
-        file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
-
-        st = os.stat("%s.log" % file)
+        st = os.stat("%s.log" % self.basename)
         return time.time() - st.st_mtime
 
     def age_ctime(self):
         """get the age of build from ctime"""
-        file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
-
-        st = os.stat("%s.log" % file)
+        st = os.stat("%s.log" % self.basename)
         return time.time() - st.st_ctime
 
     def read_log(self):
         """read full log file"""
-        return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".log", "r")
+        return open(self.basename+".log", "r")
 
     def read_err(self):
         """read full err file"""
         try:
-            return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".err", 'r')
+            return open(self.basename+".err", 'r')
         except IOError:
             # No such file
             return StringIO()
@@ -232,7 +232,7 @@ class Build(object):
         timestamp = None
         f = self.read_log()
         try:
-            for l in f.readlines():
+            for l in f:
                 if l.startswith("BUILD COMMIT REVISION: "):
                     commit_revid = l.split(":", 1)[1].strip()
                 elif l.startswith("BUILD REVISION: "):
@@ -270,9 +270,8 @@ class CachingBuild(Build):
     to calculate."""
 
     def revision_details(self):
-        file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
         cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
-        st1 = os.stat("%s.log" % file)
+        st1 = os.stat("%s.log" % self.basename)
 
         try:
             st2 = os.stat("%s.revision" % cachef)
@@ -297,9 +296,8 @@ class CachingBuild(Build):
         return (revid, commit_revid, timestamp)
 
     def err_count(self):
-        file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
         cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
-        st1 = os.stat("%s.err" % file)
+        st1 = os.stat("%s.err" % self.basename)
 
         try:
             st2 = os.stat("%s.errcount" % cachef)
@@ -318,10 +316,9 @@ class CachingBuild(Build):
         return ret
 
     def status(self):
-        file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
         cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)+".status"
 
-        st1 = os.stat("%s.log" % file)
+        st1 = os.stat("%s.log" % self.basename)
 
         try:
             st2 = os.stat(cachefile)
@@ -433,6 +430,21 @@ class BuildResultStore(object):
 
         return ret
 
+    def upload_build(self, build):
+        (rev, commit_rev, rev_timestamp) = build.revision_details()
+
+        if commit_rev is not None:
+            rev = commit_rev
+
+        new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
+        os.rename(build.basename+".log", new_basename+".log")
+        if os.path.exists(build.basename+".err"):
+            os.rename(build.basename+".err", new_basename+".err")
+
+        # FIXME:
+        # $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+        # $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
+
 """
     def get_previous_revision(self, tree, host, compiler, revision):
         # Look up the database to find the previous status
@@ -443,60 +455,6 @@ class BuildResultStore(object):
             $old_status_html = @row[0]
             $old_rev = @row[1]
             $old_commit = @row[2]
-
-    def upload_build(self, build):
-
-        my $expression = "SELECT checksum FROM build WHERE age >= ? AND tree = ? AND host = ? AND compiler = ?"
-        my $st = $dbh->prepare($expression)
-        $st->execute($stat->ctime, $tree, $host, $compiler)
-        # Don't bother if we've already processed this file
-        my $relevant_rows = $st->fetchall_arrayref()
-        $st->finish()
-
-        if relevant_rows > 0:
-            return
-
-        data = build.read_log()
-        # Don't bother with empty logs, they have no meaning (and would all share the same checksum)
-        if not data:
-            return
-
-        err = build.read_err()
-        checksum = build.log_checksum()
-        if ($dbh->selectrow_array("SELECT checksum FROM build WHERE checksum = '$checksum'")):
-            $dbh->do("UPDATE BUILD SET age = ? WHERE checksum = ?", undef, 
-                 ($stat->ctime, $checksum))
-            continue
-
-        (rev, rev_timestamp) = build.revision_details()
-
-        status_html = db.build_status_from_logs(data, err)
-
-        $st->finish()
-
-        $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
-        $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
-
-       $st->finish()
-
-        cur_status = db.build_status_info_from_html(rev, commit, status_html)
-
-        # If we were able to put this into the DB (ie, a
-        # one-off event, so we won't repeat this), then also
-        # hard-link the log files to the revision, if we know
-        # it.
-
-        # This ensures that the names under 'oldrev' are well known and well formed 
-        log_rev = self.build_fname(tree, host, compiler, commit) + ".log"
-        err_rev = self.build_fname(tree, host, compiler, commit) + ".err"
-        unlink $log_rev
-        unlink $err_rev
-        link($logfn . ".log", $log_rev) || die "Failed to link $logfn to $log_rev"
-
-        # this prevents lots of links building up with err files
-        copy($logfn . ".err", $err_rev) || die "Failed to copy $logfn to $err_rev"
-        unlink($logfn . ".err")
-        link($err_rev, $logfn . ".err")
         """
 
 
diff --git a/buildfarm/util.py b/buildfarm/util.py
index 60b263e..3b18164 100644
--- a/buildfarm/util.py
+++ b/buildfarm/util.py
@@ -67,6 +67,7 @@ def FileSave(filename, contents):
     finally:
         f.close()
 
+
 def ChangeExtension(fname, ext):
     """return a filename with a changed extension"""
     try:
@@ -83,8 +84,6 @@ def dhm_time(sec):
     hour = int(sec / (60*60)) % 24;
     min = int(sec / 60) % 60;
 
-    ret = ""
-
     if sec < 0:
         return "-"
 
diff --git a/import-and-analyse.py b/import-and-analyse.py
index d45be19..de7f08d 100644
--- a/import-and-analyse.py
+++ b/import-and-analyse.py
@@ -12,7 +12,6 @@ on recent commits.
 
 from buildfarm import (
     BuildFarm,
-    data,
     hostdb,
     )
 import commands
@@ -23,8 +22,6 @@ import os
 import re
 import smtplib
 
-dry_run = True
-
 parser = optparse.OptionParser("import-and-analyse [options]")
 parser.add_option("--dry-run", help="Will cause the script to send output to stdout instead of to sendmail.", action="store_true")
 parser.add_option("--verbose", help="Be verbose", action="count")
@@ -35,7 +32,7 @@ UNPACKED_DIR = "/home/ftp/pub/unpacked"
 
 # we open readonly here as only apache(www-run) has write access
 buildfarm = BuildFarm()
-db = data.BuildResultStore(os.path.abspath(os.path.dirname(__file__)), True)
+db = buildfarm.builds
 hostsdb = buildfarm.hostdb
 
 smtp = smtplib.SMTP()
@@ -105,18 +102,18 @@ def check_and_send_mails(tree, host, compiler, cur, old):
     (old_rev, old_rev_timestamp) = old.revision_details()
     old_status = old.status()
 
-    if dry_run:
+    if opts.dry_run:
         print "rev=%s status=%s" % (cur_rev, cur_status)
         print "old rev=%s status=%s" % (old_rev, old_status)
 
     if not cur_status.regressed_since(old_status):
-        if dry_run:
+        if opts.dry_run:
             print "the build didn't get worse since %r" % old_status
         return
 
     log = get_log(tree, cur, old)
     if not log:
-        if dry_run:
+        if opts.dry_run:
             print "no log"
         return
 
@@ -135,7 +132,8 @@ See http://build.samba.org/?function=View+Build;host=%(host)s;tree=%(tree)s;comp
 The build may have been broken by one of the following commits:
 
 %(change_log)s
-    """ % {"tree": tree, "host": host, "compiler": compiler, "change_log": log.change_log, "scm": t.scm, "branch": t.branch}
+    """ % {"tree": tree, "host": host, "compiler": compiler, "change_log": log.change_log, "scm": t.scm, "branch": t.branch,
+            "cur_rev": cur_rev, "old_rev": old_rev, "cur_status": cur_status, "old_status": old_status }
 
     msg = MIMEText(body)
     msg["Subject"] = "BUILD of %s:%s BROKEN on %s with %s AT REVISION %s" % (tree, t.branch, host, compiler, cur_rev)
@@ -150,7 +148,7 @@ for build in buildfarm.get_new_builds():
 
     db.upload_build(build)
 
-    (rev, commit_rev, rev_timestamp) = db.revision_details()
+    (rev, commit_rev, rev_timestamp) = build.revision_details()
 
     try:
         prev_rev = db.get_previous_revision(build.tree, build.host, build.compiler, rev)
@@ -159,7 +157,7 @@ for build in buildfarm.get_new_builds():
         continue
     else:
         prev_build = db.get_build(build.tree, build.host, build.compiler, prev_rev)
-        check_and_send_mails(build.tree, build.host, build.compiler, build.status(), prev_build.status())
+        check_and_send_mails(build.tree, build.host, build.compiler, build, prev_build)
 
 
 smtp.quit()


-- 
build.samba.org


More information about the samba-cvs mailing list