[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Tue Nov 2 02:04:02 MDT 2010
The branch, master has been updated
via d273a3a Add more tests, consistent parameter ordering.
from 1bfc40b Add configuration file with tree data.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit d273a3a78192b4544dd508ee4cbdfe4264c05e05
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Tue Nov 2 09:03:51 2010 +0100
Add more tests, consistent parameter ordering.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/data.py | 68 ++++++++++++++++++++++++-----------
buildfarm/tests/__init__.py | 17 ++++++++-
buildfarm/tests/test_data.py | 31 ++++++++++++++--
web/build.py | 81 ++++++++++++++++++++---------------------
4 files changed, 130 insertions(+), 67 deletions(-)
Changeset truncated at 500 lines:
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 03b206b..8458c58 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -38,6 +38,15 @@ def check_dir_exists(kind, path):
raise Exception("%s directory %s does not exist" % (kind, path))
+class NoSuchBuildError(Exception):
+ """The build with the specified name does not exist."""
+
+ def __init__(self, tree, host, compiler, rev=None):
+ self.tree = tree
+ self.host = host
+ self.compiler = compiler
+ self.rev = rev
+
def status_info_cmp(self, s1, s2):
a1 = s1["array"]
@@ -74,6 +83,7 @@ class Tree(object):
def read_trees_from_conf(path):
+ """Read trees from a configuration file."""
ret = {}
cfp = ConfigParser.ConfigParser()
cfp.readfp(open(path))
@@ -82,7 +92,7 @@ def read_trees_from_conf(path):
return s
-class BuildfarmDatabase(object):
+class BuildResultStore(object):
"""The build farm build result database."""
OLDAGE = 60*60*4,
@@ -133,30 +143,30 @@ class BuildfarmDatabase(object):
# on a host.
# the ctime age is used to determine when the last real build happened
- def build_age_mtime(self, host, tree, compiler, rev):
+ def build_age_mtime(self, tree, host, compiler, rev=None):
"""get the age of build from mtime"""
- file = self.build_fname(tree, host, compiler, rev)
+ file = self.build_fname(tree, host, compiler, rev=rev)
try:
st = os.stat("%s.log" % file)
except OSError:
# File does not exist
- return -1
+ raise NoSuchBuildError(tree, host, compiler, rev)
else:
return time.time() - st.st_mtime
- def build_age_ctime(self, host, tree, compiler, rev):
+ def build_age_ctime(self, tree, host, compiler, rev=None):
"""get the age of build from ctime"""
- file = self.build_fname(tree, host, compiler, rev)
+ file = self.build_fname(tree, host, compiler, rev=rev)
try:
st = os.stat("%s.log" % file)
except OSError:
- return -1
+ raise NoSuchBuildError(tree, host, compiler, rev)
else:
return time.time() - st.st_ctime
- def build_revision_details(self, host, tree, compiler, rev=None):
+ def build_revision_details(self, tree, host, compiler, rev=None):
"""get the svn revision of build"""
file = self.build_fname(tree, host, compiler, rev)
cachef = self.cache_fname(tree, host, compiler, rev)
@@ -207,12 +217,12 @@ class BuildfarmDatabase(object):
return ret
- def build_revision(self, host, tree, compiler, rev):
- r = self.build_revision_details(host, tree, compiler, rev)
+ def build_revision(self, tree, host, compiler, rev=None):
+ r = self.build_revision_details(tree, host, compiler, rev=rev)
return r.split(":")[0]
- def build_revision_time(self, host, tree, compiler, rev):
- r = self.build_revision_details(host, tree, compiler, rev)
+ def build_revision_time(self, tree, host, compiler, rev=None):
+ r = self.build_revision_details(tree, host, compiler, rev)
return r.split(":", 1)[1]
def build_status_from_logs(self, log, err):
@@ -279,11 +289,11 @@ class BuildfarmDatabase(object):
return "%s/%s/%s/%s%s%s%s" % (
cstatus, bstatus, istatus, tstatus, sstatus, dstatus, tostatus)
- def build_status(self, host, tree, compiler, rev):
+ def build_status(self, tree, host, compiler, rev=None):
"""get status of build
- :param host: Host name
:param tree: Tree name
+ :param host: Host name
:param compiler: Compiler name
:param rev: Revision
:return: string with build status
@@ -366,18 +376,19 @@ class BuildfarmDatabase(object):
status_raw = util.strip_html(status_html)
return self.build_status_info_from_string(rev_seq, rev, status_raw)
- def build_status_info(self, host, tree, compiler, rev_seq):
- """find the build status as an perl object
+ def build_status_info(self, tree, host, compiler, rev_seq):
+ """find the build status as an object
the 'value' gets one point for passing each stage
"""
- rev = self.build_revision(host, tree, compiler, rev_seq)
- status_html = self.build_status(host, tree, compiler, rev_seq)
+ rev = self.build_revision(tree, host, compiler, rev_seq)
+ status_html = self.build_status(tree, host, compiler, rev_seq)
return self.build_status_info_from_html(rev_seq, rev, status_html)
def lcov_status(self, tree):
"""get status of build"""
- cachefile = os.path.join(self.cachedir, "lcov.%s.%s.status" % (self.LCOVHOST, tree))
+ cachefile = os.path.join(self.cachedir, "lcov.%s.%s.status" % (
+ self.LCOVHOST, tree))
file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
try:
st1 = os.stat(file)
@@ -403,7 +414,7 @@ class BuildfarmDatabase(object):
util.FileSave(cachefile, ret)
return ret
- def err_count(self, host, tree, compiler, rev):
+ def err_count(self, tree, host, compiler, rev):
"""get status of build"""
file = self.build_fname(tree, host, compiler, rev)
cachef = self.cache_fname(tree, host, compiler, rev)
@@ -445,6 +456,7 @@ class BuildfarmDatabase(object):
def get_old_revs(self, tree, host, compiler):
"""get a list of old builds and their status."""
+ ret = []
directory = os.path.join(self.datadir, "oldrevs")
logfiles = [d for d in os.listdir(directory) if d.startswith("build.%s.%s.%s-" % (tree, host, compiler)) and d.endswith(".log")]
for l in logfiles:
@@ -456,7 +468,7 @@ class BuildfarmDatabase(object):
if stat.st_nlink == 2:
continue
r = {
- "STATUS": self.build_status(host, tree, compiler, rev),
+ "STATUS": self.build_status(tree, host, compiler, rev),
"REVISION": rev,
"TIMESTAMP": stat.st_ctime
}
@@ -468,3 +480,17 @@ class BuildfarmDatabase(object):
def has_host(self, host):
return host in os.listdir(os.path.join(self.datadir, "upload"))
+
+ def host_age(self, host):
+ """get the overall age of a host"""
+ ret = -1
+ for compiler in self.compilers:
+ for tree in self.trees:
+ try:
+ age = self.db.build_age_mtime(tree, host, compiler)
+ except NoSuchBuildError:
+ pass
+ else:
+ if (age < ret or ret == -1):
+ ret = age
+ return ret
diff --git a/buildfarm/tests/__init__.py b/buildfarm/tests/__init__.py
index b304bcc..ec4f3fc 100644
--- a/buildfarm/tests/__init__.py
+++ b/buildfarm/tests/__init__.py
@@ -25,6 +25,21 @@ class BuildFarmTestCase(TestCase):
"""Test case class that provides a build farm data directory and convenience methods.
"""
+ def create_mock_logfile(self, tree, host, compiler, rev=None):
+ basename = "build.%s.%s.%s" % (tree, host, compiler)
+ if rev:
+ basename += "-%s" % rev
+ path = os.path.join(self.path, "data", "oldrevs", basename + "-%s" % rev)
+ else:
+ path = os.path.join(self.path, "data", "upload", basename)
+ path += ".log"
+ f = open(path, 'w+')
+ try:
+ f.write("foo\n")
+ finally:
+ f.close()
+ return path
+
def write_hosts(self, hosts):
f = open(os.path.join(self.path, "web", "hosts.list"), "w")
try:
@@ -56,7 +71,7 @@ class BuildFarmTestCase(TestCase):
super(BuildFarmTestCase, self).setUp()
self.path = tempfile.mkdtemp()
- for subdir in ["data", "cache", "web", "lcov", "lcov/data"]:
+ for subdir in ["data", "data/upload", "data/oldrevs", "cache", "web", "lcov", "lcov/data"]:
os.mkdir(os.path.join(self.path, subdir))
def tearDown(self):
diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py
index c178fa5..778620f 100755
--- a/buildfarm/tests/test_data.py
+++ b/buildfarm/tests/test_data.py
@@ -15,6 +15,8 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+import os
+import time
import unittest
from buildfarm import data
@@ -26,19 +28,19 @@ class NonexistantTests(unittest.TestCase):
def test_nonexistant(self):
self.assertRaises(
- Exception, data.BuildfarmDatabase, "somedirthatdoesn'texist", None)
+ Exception, data.BuildResultStore, "somedirthatdoesn'texist", None)
-class BuildfarmDatabaseTests(BuildFarmTestCase):
+class BuildResultStoreTests(BuildFarmTestCase):
def setUp(self):
- super(BuildfarmDatabaseTests, self).setUp()
+ super(BuildResultStoreTests, self).setUp()
self.write_compilers(["cc"])
self.write_hosts(["gwenhwyvar", "charis"])
self.write_trees({"tdb": {"scm": "git", "repo": "tdb", "branch": "master"}})
- self.x = data.BuildfarmDatabase(self.path)
+ self.x = data.BuildResultStore(self.path)
def test_build_fname(self):
self.assertEquals(
@@ -55,3 +57,24 @@ class BuildfarmDatabaseTests(BuildFarmTestCase):
self.assertEquals(
self.x.cache_fname("mytree", "myhost", "cc"),
"%s/cache/build.mytree.myhost.cc" % self.path)
+
+ def test_build_age_mtime(self):
+ path = self.create_mock_logfile("tdb", "charis", "cc")
+ # Set mtime to something in the past
+ os.utime(path, (time.time(), time.time() - 990))
+ age = self.x.build_age_mtime("tdb", "charis", "cc")
+ self.assertTrue(age >= 990 and age <= 1000, "age was %d" % age)
+
+ def test_build_age_mtime_nonexistant(self):
+ self.assertRaises(data.NoSuchBuildError, self.x.build_age_mtime, "tdb",
+ "charis", "cc")
+
+ def test_build_age_ctime(self):
+ path = self.create_mock_logfile("tdb", "charis", "cc")
+ # Set mtime to something in the past
+ age = self.x.build_age_ctime("tdb", "charis", "cc")
+ self.assertTrue(age >= 0 and age <= 10, "age was %d" % age)
+
+ def test_build_age_ctime_nonexistant(self):
+ self.assertRaises(data.NoSuchBuildError, self.x.build_age_ctime, "tdb",
+ "charis", "cc")
diff --git a/web/build.py b/web/build.py
index ec5b318..a17eba6 100755
--- a/web/build.py
+++ b/web/build.py
@@ -39,7 +39,7 @@ import wsgiref.util
webdir = os.path.dirname(__file__)
basedir = os.path.abspath(os.path.join(webdir, ".."))
-db = data.BuildfarmDatabase(basedir)
+db = data.BuildResultStore(basedir)
history = history.History(db)
compilers = db.compilers
@@ -69,28 +69,18 @@ def get_param(form, param):
return result[0]
-def build_link(myself, host, tree, compiler, rev, status):
+def build_link(myself, tree, host, compiler, rev, status):
if rev:
opt_rev = ';revision=%s' % rev
else:
opt_rev = ''
- return "<a href='%s?function=View+Build;host=%s;tree=%s;compiler=%s%s'>%s</a>" % (myself, host, tree, compiler, opt_rev, status)
+ return "<a href='%s?function=View+Build;host=%s;tree=%s;compiler=%s%s'>%s</a>" % (
+ myself, tree, host, compiler, opt_rev, status)
-def build_status(myself, host, tree, compiler, rev):
- status = db.build_status(host, tree, compiler, rev)
- return build_link(myself, host, tree, compiler, rev, status)
-
-
-def host_age(host):
- """get the overall age of a host"""
- ret = -1
- for compiler in compilers:
- for tree in trees:
- age = db.build_age_mtime(host, tree, compiler, "")
- if age != -1 and (age < ret or ret == -1):
- ret = age
- return ret
+def build_status(myself, tree, host, compiler, rev):
+ status = db.build_status(tree, host, compiler, rev)
+ return build_link(myself, tree, host, compiler, rev, status)
def red_age(age):
@@ -139,12 +129,14 @@ def view_summary(myself, output_type):
for host in hosts:
for compiler in compilers:
for tree in trees:
- status = build_status(myself, host, tree, compiler, "")
+ status = build_status(myself, tree, host, compiler, "")
if status.startswith("Unknown Build"):
continue
- age_mtime = db.build_age_mtime(host, tree, compiler, "")
-
- if age_mtime != -1:
+ try:
+ age_mtime = db.build_age_mtime(tree, host, compiler)
+ except data.NoSuchBuildError:
+ pass
+ else:
host_count[tree]+=1
if "status failed" in status:
@@ -248,12 +240,15 @@ def view_recent_builds(myself, tree, sort_by):
for host in hosts:
for compiler in compilers:
- status = build_status(myself, host, tree, compiler, "")
- age_mtime = db.build_age_mtime(host, tree, compiler, "")
- age_ctime = db.build_age_ctime(host, tree, compiler, "")
- revision = db.build_revision(host, tree, compiler, "")
- revision_time = db.build_revision_time(host, tree, compiler, "")
- if age_mtime != -1:
+ status = build_status(myself, tree, host, compiler)
+ try:
+ age_mtime = db.build_age_mtime(tree, host, compiler)
+ except data.NoSuchBuildError:
+ pass
+ else:
+ age_ctime = db.build_age_ctime(tree, host, compiler)
+ revision = db.build_revision(tree, host, compiler)
+ revision_time = db.build_revision_time(tree, host, compiler)
all_builds.append([age_ctime, hosts[host], "<a href='%s?function=View+Host;host=%s;tree=%s;compiler=%s#%s'>%s</a>" % (myself, host, tree, compiler, host, host), compiler, tree, status, revision_link(myself, revision, tree), revision_time])
all_builds.sort(cmp_funcs[sort_by])
@@ -306,7 +301,7 @@ def draw_dead_hosts(output_type, *deadhosts):
yield "<tbody>"
for host in deadhosts:
- age_ctime = host_age(host)
+ age_ctime = db.host_age(host)
yield "<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (host, hosts[host], util.dhm_time(age_ctime))
yield "</tbody></table>"
@@ -334,7 +329,7 @@ def show_oldrevs(myself, tree, host, compiler):
if s == lastrev:
continue
lastrev = s
- ret+= "<tr><td>%s</td><td>%s</td></tr>" % (revision_link(myself, revision, tree), build_link(myself, host, tree, compiler, rev["REVISION"], rev["STATUS"]))
+ ret+= "<tr><td>%s</td><td>%s</td></tr>" % (revision_link(myself, revision, tree), build_link(myself, tree, host, compiler, rev["REVISION"], rev["STATUS"]))
if lastrev != "":
# Only print table if there was any actual data
@@ -351,9 +346,9 @@ def view_build(myself, tree, host, compiler, rev, plain_logs=False):
uname = ""
cflags = ""
config = ""
- age_mtime = db.build_age_mtime(host, tree, compiler, rev)
- revision = db.build_revision(host, tree, compiler, rev)
- status = build_status(myself, host, tree, compiler, rev)
+ age_mtime = db.build_age_mtime(tree, host, compiler, rev)
+ revision = db.build_revision(tree, host, compiler, rev)
+ status = build_status(myself, tree, host, compiler, rev)
assert re.match("^[0-9a-fA-F]*$", rev)
@@ -459,12 +454,15 @@ def view_host(myself, output_type, *requested_hosts):
for compiler in compilers:
for tree in sorted(trees.keys()):
- revision = db.build_revision(host, tree, compiler, "")
- age_mtime = db.build_age_mtime(host, tree, compiler, "")
- age_ctime = db.build_age_ctime(host, tree, compiler, "")
- warnings = db.err_count(host, tree, compiler, "")
- if age_ctime != -1:
- status = build_status(myself, host, tree, compiler, "")
+ revision = db.build_revision(tree, host, compiler)
+ try:
+ age_mtime = db.build_age_mtime(tree, host, compiler)
+ except data.NoSuchBuildError:
+ pass
+ else:
+ age_ctime = db.build_age_ctime(tree, host, compiler)
+ warnings = db.err_count(tree, host, compiler)
+ status = build_status(myself, tree, host, compiler, "")
if row == 0:
if output_type == 'text':
yield "%-12s %-10s %-10s %-10s %-10s\n" % (
@@ -486,7 +484,7 @@ def view_host(myself, output_type, *requested_hosts):
yield "<td><span class='tree'>" + tree_link(myself, tree) +"</span>/" + compiler + "</td>"
yield "<td>" + revision_link(myself, revision, tree) + "</td>"
yield "<td><div class='age'>" + red_age(age_mtime) + "</div></td>"
- yield "<td><div class='status'>" + status + "</div></td>"
+ yield "<td><div class='status'>%s</div></td>" % status
yield "<td>%s</td>" % warnings
yield "</tr>"
row+=1
@@ -521,7 +519,6 @@ def subunit_to_buildfarm_result(subunit_result):
def format_subunit_reason(reason):
reason = re.sub("^\[\n+(.*?)\n+\]$", "\\1", reason)
-
return "<div class=\"reason\">%s</div>" % reason
@@ -592,12 +589,14 @@ def print_log_cc_checker(input):
output = ""
# for now, we only handle the IBM Checker's output style
- if not m.search("^BEAM_VERSION", input):
+ if not re.search("^BEAM_VERSION", input):
return "here"
return input
content = ""
inEntry = 0
+ title = None
+ status = None
for line in input.splitlines():
# for each line, check if the line is a new entry,
--
build.samba.org
More information about the samba-cvs
mailing list