[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Wed Nov 10 01:32:36 MST 2010
The branch, master has been updated
via 7851c49 Split up further.
via 2f489ce Split up BuildResultStore further.
via 1300d88 Move more functionality to buildfarm.BuildFarm.
via cea251c Remove trees from BuildResultStore.
via ecd6140 Add BuildFarm object, remove compilers and host_age from BuildResultStore.
via 0669b47 Fix smtplib.Connection -> smtplib.SMTP.
via 69c7574 Add python version of import-and-analyse.pl.
from 6b3372f Remove unused variable.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 7851c49eaa94c31e8e89cef7d5551722bbb4e451
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 09:31:49 2010 +0100
Split up further.
commit 2f489ce85384cc5a81237187db1f39dac12d771b
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 09:26:07 2010 +0100
Split up BuildResultStore further.
commit 1300d8845dd004d73d8c0e2cc780093ba442b8c6
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 08:32:38 2010 +0100
Move more functionality to buildfarm.BuildFarm.
commit cea251c95ae5c166f6b40da896f3b395d1d3ca6a
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 08:23:49 2010 +0100
Remove trees from BuildResultStore.
commit ecd61407c8e7c6e59a18c810a3038493f54e37e9
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 08:19:01 2010 +0100
Add BuildFarm object, remove compilers and host_age from BuildResultStore.
commit 0669b471176c6204afa1cb75b37e7c7d314f690e
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 08:09:05 2010 +0100
Fix smtplib.Connection -> smtplib.SMTP.
commit 69c7574774c799c6a2ee3c11b6dd51640e1b1eb7
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 08:08:48 2010 +0100
Add python version of import-and-analyse.pl.
-----------------------------------------------------------------------
Summary of changes:
admin.py | 6 +-
buildfarm/__init__.py | 107 ++++++++++++++++-
buildfarm/data.py | 274 +++++++++++++++++++++++-------------------
buildfarm/tests/__init__.py | 69 ++++++++++-
buildfarm/tests/test_data.py | 122 +++++++------------
import-and-analyse.py | 181 ++++++++++++++++++++++++++++
mail-dead-hosts.py | 8 +-
web/build.py | 17 ++--
8 files changed, 562 insertions(+), 222 deletions(-)
create mode 100644 import-and-analyse.py
Changeset truncated at 500 lines:
diff --git a/admin.py b/admin.py
index d94b572..909d303 100755
--- a/admin.py
+++ b/admin.py
@@ -19,7 +19,7 @@
from buildfarm import (
hostdb,
- open_hostdb,
+ BuildFarm,
)
import commands
import os
@@ -28,7 +28,9 @@ import sys
import time
from email.MIMEText import MIMEText
-db = open_hostdb()
+buildfarm = BuildFarm()
+
+db = buildfarm.hostdb
dry_run = False
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index e133cda..30c6005 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -17,9 +17,108 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+import ConfigParser
import os
-def open_hostdb():
- from buildfarm import hostdb
- return hostdb.HostDatabase(
- os.path.join(os.path.dirname(__file__), "..", "hostdb.sqlite"))
+
+class Tree(object):
+ """A tree to build."""
+
+ def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
+ self.name = name
+ self.repo = repo
+ self.scm = scm
+ self.branch = branch
+ self.subdir = subdir
+ self.srcdir = srcdir
+ self.scm = scm
+
+ def __repr__(self):
+ return "<%s %r>" % (self.__class__.__name__, self.name)
+
+
+def read_trees_from_conf(path):
+ """Read trees from a configuration file."""
+ ret = {}
+ cfp = ConfigParser.ConfigParser()
+ cfp.readfp(open(path))
+ for s in cfp.sections():
+ ret[s] = Tree(name=s, **dict(cfp.items(s)))
+ return ret
+
+
+def lcov_extract_percentage(text):
+ m = re.search('\<td class="headerItem".*?\>Code\ \;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', text)
+ if m:
+ return m.group(1)
+ else:
+ return None
+
+
+class BuildFarm(object):
+
+ LCOVHOST = "magni"
+ OLDAGE = 60*60*4,
+ DEADAGE = 60*60*24*4
+
+ def __init__(self, path=None):
+ if path is None:
+ path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ self.path = path
+ self.webdir = os.path.join(self.path, "web")
+ if not os.path.isdir(path):
+ raise Exception("web directory %s does not exist" % self.webdir)
+ self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
+ self.builds = self._open_build_results()
+ self.hostdb = self._open_hostdb()
+ self.compilers = self._load_compilers()
+ self.lcovdir = os.path.join(self.path, "lcov/data")
+
+ def __repr__(self):
+ return "%s(%r)" % (self.__class__.__name__, self.path)
+
+ def _open_build_results(self):
+ from buildfarm import data
+ return data.BuildResultStore(os.path.join(self.path, "data", "oldrevs"))
+
+ def _open_hostdb(self):
+ from buildfarm import hostdb
+ return hostdb.HostDatabase(
+ os.path.join(self.path, "hostdb.sqlite"))
+
+ def _load_compilers(self):
+ from buildfarm import util
+ return util.load_list(os.path.join(self.webdir, "compilers.list"))
+
+ def lcov_status(self, tree):
+ """get status of build"""
+ from buildfarm import data, util
+ cachefile = os.path.join(self.builds.cachedir, "lcov.%s.%s.status" % (
+ self.LCOVHOST, tree))
+ file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
+ try:
+ st1 = os.stat(file)
+ except OSError:
+ # File does not exist
+ raise data.NoSuchBuildError(tree, self.LCOVHOST, "lcov")
+ try:
+ st2 = os.stat(cachefile)
+ except OSError:
+ # file does not exist
+ st2 = None
+
+ if st2 and st1.st_ctime <= st2.st_mtime:
+ ret = util.FileLoad(cachefile)
+ if ret == "":
+ return None
+ return ret
+
+ lcov_html = util.FileLoad(file)
+ perc = lcov_extract_percentage(lcov_html)
+ if perc is None:
+ ret = ""
+ else:
+ ret = perc
+ if self.readonly:
+ util.FileSave(cachefile, ret)
+ return perc
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 2d03f2c..7ff8c92 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -22,7 +22,6 @@
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-import ConfigParser
from cStringIO import StringIO
import hashlib
import os
@@ -31,6 +30,16 @@ import time
import util
+class BuildSummary(object):
+
+ def __init__(self, host, tree, compiler, rev, status):
+ self.host = host
+ self.tree = tree
+ self.compiler = compiler
+ self.rev = rev
+ self.status = status
+
+
class BuildStatus(object):
def __init__(self, stages=None, other_failures=None):
@@ -146,14 +155,6 @@ def build_status_from_logs(log, err):
return ret
-def lcov_extract_percentage(text):
- m = re.search('\<td class="headerItem".*?\>Code\ \;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', text)
- if m:
- return m.group(1)
- else:
- return None
-
-
class NoSuchBuildError(Exception):
"""The build with the specified name does not exist."""
@@ -164,22 +165,6 @@ class NoSuchBuildError(Exception):
self.rev = rev
-class Tree(object):
- """A tree to build."""
-
- def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
- self.name = name
- self.repo = repo
- self.scm = scm
- self.branch = branch
- self.subdir = subdir
- self.srcdir = srcdir
- self.scm = scm
-
- def __repr__(self):
- return "<%s %r>" % (self.__class__.__name__, self.name)
-
-
class Build(object):
"""A single build of a tree on a particular host using a particular compiler.
"""
@@ -222,7 +207,6 @@ class Build(object):
# No such file
return StringIO()
-
def log_checksum(self):
f = self.read_log()
try:
@@ -230,6 +214,13 @@ class Build(object):
finally:
f.close()
+ def summary(self):
+ (revid, commit_revid, timestamp) = self.revision_details()
+ if commit_revid:
+ revid = commit_revid
+ status = self.status()
+ return BuildSummary(self.host, self.tree, self.compiler, revid, status)
+
def revision_details(self):
"""get the revision of build
@@ -357,109 +348,84 @@ class CachingBuild(Build):
return ret
-def read_trees_from_conf(path):
- """Read trees from a configuration file."""
- ret = {}
- cfp = ConfigParser.ConfigParser()
- cfp.readfp(open(path))
- for s in cfp.sections():
- ret[s] = Tree(name=s, **dict(cfp.items(s)))
- return ret
+class UploadBuildResultStore(object):
+ def __init__(self, path):
+ """Open the database.
-class BuildResultStore(object):
- """The build farm build result database."""
+ :param path: Build result base directory
+ """
+ self.path = path
+
+ def build_fname(self, tree, host, compiler):
+ return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
+
+ def has_host(self, host):
+ for name in os.listdir(self.path):
+ try:
+ if name.split(".")[2] == host:
+ return True
+ except IndexError:
+ pass
+ return False
+
+ def get_build(self, tree, host, compiler):
+ logf = self.build_fname(tree, host, compiler) + ".log"
+ if not os.path.exists(logf):
+ raise NoSuchBuildError(tree, host, compiler)
+ return Build(self, tree, host, compiler)
- OLDAGE = 60*60*4,
- DEADAGE = 60*60*24*4
- LCOVHOST = "magni"
- def __init__(self, basedir, readonly=False):
+class CachingUploadBuildResultStore(UploadBuildResultStore):
+
+ def __init__(self, basedir, cachedir, readonly=False):
"""Open the database.
- :param basedir: Build result base directory
:param readonly: Whether to avoid saving cache files
"""
- self.basedir = basedir
- check_dir_exists("base", self.basedir)
+ super(CachingUploadBuildResultStore, self).__init__(basedir)
+ self.cachedir = cachedir
self.readonly = readonly
- self.webdir = os.path.join(basedir, "web")
- check_dir_exists("web", self.webdir)
+ def cache_fname(self, tree, host, compiler):
+ return os.path.join(self.cachedir, "build.%s.%s.%s" % (tree, host, compiler))
- self.datadir = os.path.join(basedir, "data")
- check_dir_exists("data", self.datadir)
+ def get_build(self, tree, host, compiler):
+ logf = self.build_fname(tree, host, compiler) + ".log"
+ if not os.path.exists(logf):
+ raise NoSuchBuildError(tree, host, compiler)
+ return CachingBuild(self, tree, host, compiler)
- self.cachedir = os.path.join(basedir, "cache")
- check_dir_exists("cache", self.cachedir)
- self.lcovdir = os.path.join(basedir, "lcov/data")
- check_dir_exists("lcov", self.lcovdir)
+class BuildResultStore(object):
+ """The build farm build result database."""
- self.compilers = util.load_list(os.path.join(self.webdir, "compilers.list"))
+ def __init__(self, path):
+ """Open the database.
- self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
+ :param path: Build result base directory
+ """
+ self.path = path
- def get_build(self, tree, host, compiler, rev=None):
+ def get_build(self, tree, host, compiler, rev):
logf = self.build_fname(tree, host, compiler, rev) + ".log"
if not os.path.exists(logf):
raise NoSuchBuildError(tree, host, compiler, rev)
- return CachingBuild(self, tree, host, compiler, rev)
+ return Build(self, tree, host, compiler, rev)
- def cache_fname(self, tree, host, compiler, rev=None):
- if rev is not None:
- return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
- else:
- return os.path.join(self.cachedir, "build.%s.%s.%s" % (tree, host, compiler))
-
- def build_fname(self, tree, host, compiler, rev=None):
+ def build_fname(self, tree, host, compiler, rev):
"""get the name of the build file"""
- if rev is not None:
- return os.path.join(self.datadir, "oldrevs/build.%s.%s.%s-%s" % (tree, host, compiler, rev))
- return os.path.join(self.datadir, "upload/build.%s.%s.%s" % (tree, host, compiler))
-
- def lcov_status(self, tree):
- """get status of build"""
- cachefile = os.path.join(self.cachedir, "lcov.%s.%s.status" % (
- self.LCOVHOST, tree))
- file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
- try:
- st1 = os.stat(file)
- except OSError:
- # File does not exist
- raise NoSuchBuildError(tree, self.LCOVHOST, "lcov")
- try:
- st2 = os.stat(cachefile)
- except OSError:
- # file does not exist
- st2 = None
-
- if st2 and st1.st_ctime <= st2.st_mtime:
- ret = util.FileLoad(cachefile)
- if ret == "":
- return None
- return ret
-
- lcov_html = util.FileLoad(file)
- perc = lcov_extract_percentage(lcov_html)
- if perc is None:
- ret = ""
- else:
- ret = perc
- if self.readonly:
- util.FileSave(cachefile, ret)
- return perc
+ return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
def get_old_revs(self, tree, host, compiler):
"""get a list of old builds and their status."""
ret = []
- directory = os.path.join(self.datadir, "oldrevs")
- logfiles = [d for d in os.listdir(directory) if d.startswith("build.%s.%s.%s-" % (tree, host, compiler)) and d.endswith(".log")]
+ logfiles = [d for d in os.listdir(self.path) if d.startswith("build.%s.%s.%s-" % (tree, host, compiler)) and d.endswith(".log")]
for l in logfiles:
m = re.match(".*-([0-9A-Fa-f]+).log$", l)
if m:
rev = m.group(1)
- stat = os.stat(os.path.join(directory, l))
+ stat = os.stat(os.path.join(self.path, l))
# skip the current build
if stat.st_nlink == 2:
continue
@@ -475,25 +441,91 @@ class BuildResultStore(object):
return ret
- def has_host(self, host):
- for name in os.listdir(os.path.join(self.datadir, "upload")):
- try:
- if name.split(".")[2] == host:
- return True
- except IndexError:
- pass
- return False
+"""
+ def get_previous_revision(self, tree, host, compiler, revision):
+ # Look up the database to find the previous status
+ $st = $dbh->prepare("SELECT status, revision, commit_revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND revision != ? AND commit_revision != ? ORDER BY id DESC LIMIT 1")
+ $st->execute( $tree, $host, $compiler, $rev, $commit)
+
+ while ( my @row = $st->fetchrow_array ) {
+ $old_status_html = @row[0]
+ $old_rev = @row[1]
+ $old_commit = @row[2]
+
+ def upload_build(self, build):
+
+ my $expression = "SELECT checksum FROM build WHERE age >= ? AND tree = ? AND host = ? AND compiler = ?"
+ my $st = $dbh->prepare($expression)
+ $st->execute($stat->ctime, $tree, $host, $compiler)
+ # Don't bother if we've already processed this file
+ my $relevant_rows = $st->fetchall_arrayref()
+ $st->finish()
+
+ if relevant_rows > 0:
+ return
+
+ data = build.read_log()
+ # Don't bother with empty logs, they have no meaning (and would all share the same checksum)
+ if not data:
+ return
+
+ err = build.read_err()
+ checksum = build.log_checksum()
+ if ($dbh->selectrow_array("SELECT checksum FROM build WHERE checksum = '$checksum'")):
+ $dbh->do("UPDATE BUILD SET age = ? WHERE checksum = ?", undef,
+ ($stat->ctime, $checksum))
+ continue
+
+ (rev, rev_timestamp) = build.revision_details()
+
+ status_html = db.build_status_from_logs(data, err)
+
+ $st->finish()
+
+ $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+ $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
+
+ $st->finish()
+
+ cur_status = db.build_status_info_from_html(rev, commit, status_html)
+
+ # If we were able to put this into the DB (ie, a
+ # one-off event, so we won't repeat this), then also
+ # hard-link the log files to the revision, if we know
+ # it.
+
+ # This ensures that the names under 'oldrev' are well known and well formed
+ log_rev = self.build_fname(tree, host, compiler, commit) + ".log"
+ err_rev = self.build_fname(tree, host, compiler, commit) + ".err"
+ unlink $log_rev
+ unlink $err_rev
+ link($logfn . ".log", $log_rev) || die "Failed to link $logfn to $log_rev"
+
+ # this prevents lots of links building up with err files
+ copy($logfn . ".err", $err_rev) || die "Failed to copy $logfn to $err_rev"
+ unlink($logfn . ".err")
+ link($err_rev, $logfn . ".err")
+ """
+
+
+class CachingBuildResultStore(BuildResultStore):
+
+ def __init__(self, basedir, cachedir, readonly=False):
+ super(CachingBuildResultStore, self).__init__(basedir)
+
+ self.cachedir = cachedir
+ check_dir_exists("cache", self.cachedir)
+
+ self.readonly = readonly
+
+ def get_build(self, tree, host, compiler, rev):
+ logf = self.build_fname(tree, host, compiler, rev) + ".log"
+ if not os.path.exists(logf):
+ raise NoSuchBuildError(tree, host, compiler, rev)
+ return CachingBuild(self, tree, host, compiler, rev)
+
+ def cache_fname(self, tree, host, compiler, rev):
+ return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
+
+
- def host_age(self, host):
- """get the overall age of a host"""
- # FIXME: Turn this into a simple SQL query, or use something in hostdb ?
- ret = None
- for compiler in self.compilers:
- for tree in self.trees:
- try:
- build = self.get_build(tree, host, compiler)
- except NoSuchBuildError:
- pass
- else:
--
build.samba.org
More information about the samba-cvs
mailing list