[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Thu Nov 18 17:23:23 MST 2010
The branch, master has been updated
via c6a5101 Add get_tree_builds.
via 44a24a7 Add newlines.
from 0195c00 Ignore some files.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit c6a5101ac133443bfb8d064b664a7c6d4be168ea
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Nov 19 01:21:35 2010 +0100
Add get_tree_builds.
commit 44a24a717f0593b008aa3df81272ec20888041c2
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Nov 19 00:05:27 2010 +0100
Add newlines.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/__init__.py | 14 ++-
buildfarm/data.py | 30 +++--
buildfarm/sqldb.py | 11 +-
buildfarm/tests/__init__.py | 8 +
buildfarm/tests/test_buildfarm.py | 32 +++-
buildfarm/web/__init__.py | 296 ++++++++++++++++++-------------------
6 files changed, 212 insertions(+), 179 deletions(-)
Changeset truncated at 500 lines:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index 5d35f42..71bf02a 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -141,10 +141,16 @@ class BuildFarm(object):
if build.tree in self.trees and build.compiler in self.compilers and build.host in hostnames:
yield build
- def get_last_builds(self, tree=None):
- for build in self.get_new_builds():
- if tree is not None and build.tree == tree:
- yield build
+ def get_last_builds(self):
+ return self.get_new_builds()
+
+ def get_tree_builds(self, tree):
+ yielded = set()
+ for build in self.builds.get_all_builds():
+ if build.tree == tree:
+ if not (build.host, build.compiler) in yielded:
+ yielded.add((build.host, build.compiler))
+ yield build
def get_host_builds(self, host):
from buildfarm import data
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 3846e8b..cf87904 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -378,22 +378,28 @@ class BuildResultStore(object):
"""get the name of the build file"""
return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
+ def get_all_builds(self):
+ for l in os.listdir(self.path):
+ m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
+ if not m:
+ continue
+ tree = m.group(1)
+ host = m.group(2)
+ compiler = m.group(3)
+ rev = m.group(4)
+ stat = os.stat(os.path.join(self.path, l))
+ # skip the current build
+ if stat.st_nlink == 2:
+ continue
+ yield self.get_build(tree, host, compiler, rev)
+
def get_old_revs(self, tree, host, compiler):
"""get a list of old builds and their status."""
ret = []
- logfiles = [d for d in os.listdir(self.path) if d.startswith("build.%s.%s.%s-" % (tree, host, compiler)) and d.endswith(".log")]
- for l in logfiles:
- m = re.match(".*-([0-9A-Fa-f]+).log$", l)
- if m:
- rev = m.group(1)
- stat = os.stat(os.path.join(self.path, l))
- # skip the current build
- if stat.st_nlink == 2:
- continue
- ret.append(self.get_build(tree, host, compiler, rev))
-
+ for build in self.get_all_builds():
+ if build.tree == tree and build.host == host and build.compiler == compiler:
+ ret.append(build)
ret.sort(lambda a, b: cmp(a.age, b.age))
-
return ret
def upload_build(self, build):
diff --git a/buildfarm/sqldb.py b/buildfarm/sqldb.py
index 4542c8a..47bfa79 100644
--- a/buildfarm/sqldb.py
+++ b/buildfarm/sqldb.py
@@ -248,11 +248,12 @@ class StormCachingBuildFarm(BuildFarm):
return self._get_store().find(StormBuild,
StormBuild.host == host).group_by(StormBuild.compiler, StormBuild.tree)
- def get_last_builds(self, tree=None):
- extra_expr = []
- if tree is not None:
- extra_expr.append(StormBuild.tree == tree)
- return self._get_store().find(StormBuild, *extra_expr)
+ def get_tree_builds(self, tree):
+ return self._get_store().find(StormBuild,
+ StormBuild.tree == tree).group_by(StormBuild.compiler, StormBuild.host)
+
+ def get_last_builds(self):
+ return self._get_store().find(StormBuild)
def commit(self):
self.store.commit()
diff --git a/buildfarm/tests/__init__.py b/buildfarm/tests/__init__.py
index b3226d5..26cbebe 100644
--- a/buildfarm/tests/__init__.py
+++ b/buildfarm/tests/__init__.py
@@ -15,6 +15,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+from buildfarm.data import Build
from buildfarm.sqldb import setup_schema
import os
from storm import database
@@ -28,6 +29,13 @@ class BuildFarmTestCase(TestCase):
"""Test case class that provides a build farm data directory and convenience methods.
"""
+ def upload_mock_logfile(self, store, tree, host, compiler, stdout_contents="", stderr_contents=None):
+ log_path = self.create_mock_logfile(tree, host, compiler, contents=stdout_contents)
+ if stderr_contents is not None:
+ err_path = self.create_mock_logfile(tree, host, compiler, kind="stderr", contents=stderr_contents)
+ build = Build(log_path[:-4], tree, host, compiler)
+ store.upload_build(build)
+
def create_mock_logfile(self, tree, host, compiler, rev=None,
kind="stdout", contents="FOO"):
basename = "build.%s.%s.%s" % (tree, host, compiler)
diff --git a/buildfarm/tests/test_buildfarm.py b/buildfarm/tests/test_buildfarm.py
index 9a62a7c..045348a 100644
--- a/buildfarm/tests/test_buildfarm.py
+++ b/buildfarm/tests/test_buildfarm.py
@@ -68,9 +68,7 @@ repo = rsync.git
branch = HEAD
""")
t = read_trees_from_conf(name)
- self.assertEquals(
- t["pidl"].scm,
- "git")
+ self.assertEquals(t["pidl"].scm, "git")
class BuildFarmTestBase(object):
@@ -78,11 +76,32 @@ class BuildFarmTestBase(object):
def setUp(self):
self.write_compilers(["cc"])
self.write_hosts({"myhost": "Fedora"})
- self.write_trees({"trivial": { "scm": "git", "repo": "git://foo", "branch": "master" }})
+ self.write_trees({"trivial": {"scm": "git", "repo": "git://foo", "branch": "master"}})
def test_get_new_builds_empty(self):
self.assertEquals([], list(self.x.get_new_builds()))
+ def test_get_last_builds_empty(self):
+ self.assertEquals([], list(self.x.get_last_builds()))
+
+ def test_get_tree_builds_empty(self):
+ self.assertEquals([], list(self.x.get_tree_builds("trival")))
+
+ def test_get_tree_builds(self):
+ path = self.upload_mock_logfile(self.x.builds, "tdb", "charis", "cc",
+ "BUILD COMMIT REVISION: 12\n")
+ path = self.upload_mock_logfile(self.x.builds, "tdb", "charis", "cc",
+ "BUILD COMMIT REVISION: 13\n")
+ path = self.upload_mock_logfile(self.x.builds, "tdb", "charis", "cc",
+ "BUILD COMMIT REVISION: 42\n")
+ builds = list(self.x.get_tree_builds("tdb"))
+ self.assertEquals(1, len(builds))
+ build = builds[0]
+ self.assertEquals("42", build.revision)
+
+ def test_get_host_builds_empty(self):
+ self.assertEquals([], list(self.x.get_host_builds("myhost")))
+
def test_lcov_status_none(self):
self.assertRaises(data.NoSuchBuildError, self.x.lcov_status, "trivial")
@@ -103,7 +122,7 @@ class BuildFarmTestBase(object):
self.assertEquals("12", build.revision)
def test_get_build_no_rev(self):
- path = self.create_mock_logfile("tdb", "charis", "cc",
+ path = self.create_mock_logfile("tdb", "charis", "cc",
contents="This is what a log file looks like.")
build = self.x.get_build("tdb", "charis", "cc")
self.assertEquals("tdb", build.tree)
@@ -118,6 +137,3 @@ class BuildFarmTests(BuildFarmTestBase, BuildFarmTestCase):
BuildFarmTestCase.setUp(self)
BuildFarmTestBase.setUp(self)
self.x = BuildFarm(self.path)
-
-
-
diff --git a/buildfarm/web/__init__.py b/buildfarm/web/__init__.py
index f33453b..591d1b0 100755
--- a/buildfarm/web/__init__.py
+++ b/buildfarm/web/__init__.py
@@ -176,19 +176,19 @@ class LogPrettyPrinter(object):
status = m.group(3)
# handle pretty-printing of static-analysis tools
if actionName == 'cc_checker':
- output = print_log_cc_checker(output)
+ output = "".join(print_log_cc_checker(output))
self.indice += 1
- return make_collapsible_html('action', actionName, output, self.indice, status)
+ return "".join(make_collapsible_html('action', actionName, output, self.indice, status))
# log is already CGI-escaped, so handle '>' in test name by handling >
def _format_stage(self, m):
self.indice += 1
- return make_collapsible_html('test', m.group(1), m.group(2), self.indice, m.group(3))
+ return "".join(make_collapsible_html('test', m.group(1), m.group(2), self.indice, m.group(3)))
def _format_skip_testsuite(self, m):
self.indice += 1
- return make_collapsible_html('test', m.group(1), '', self.indice, 'skipped')
+ return "".join(make_collapsible_html('test', m.group(1), '', self.indice, 'skipped'))
def _format_testsuite(self, m):
testName = m.group(1)
@@ -199,11 +199,11 @@ class LogPrettyPrinter(object):
else:
errorReason = ""
self.indice += 1
- return make_collapsible_html('test', testName, content+errorReason, self.indice, status)
+ return "".join(make_collapsible_html('test', testName, content+errorReason, self.indice, status))
def _format_test(self, m):
self.indice += 1
- return make_collapsible_html('test', m.group(1), m.group(2)+format_subunit_reason(m.group(4)), self.indice, subunit_to_buildfarm_result(m.group(3)))
+ return "".join(make_collapsible_html('test', m.group(1), m.group(2)+format_subunit_reason(m.group(4)), self.indice, subunit_to_buildfarm_result(m.group(3))))
def pretty_print(self, log):
# do some pretty printing for the actions
@@ -240,12 +240,11 @@ def print_log_pretty(log):
def print_log_cc_checker(input):
# generate pretty-printed html for static analysis tools
- output = ""
-
# for now, we only handle the IBM Checker's output style
if not re.search("^BEAM_VERSION", input):
- return "here"
- return input
+ yield "here"
+ # yield input
+ return
content = ""
inEntry = False
@@ -259,9 +258,9 @@ def print_log_cc_checker(input):
if line.startswith("-- "):
# got a new entry
if inEntry:
- output += make_collapsible_html('cc_checker', title, content, id, status)
+ yield "".join(make_collapsible_html('cc_checker', title, content, id, status))
else:
- output += content
+ yield content
# clear maintenance vars
(inEntry, content) = (True, "")
@@ -273,7 +272,7 @@ def print_log_cc_checker(input):
(title, status, id) = ("%s %s" % (m.group(1), m.group(4)), m.group(2), m.group(3))
elif line.startswith("CC_CHECKER STATUS"):
if inEntry:
- output += make_collapsible_html('cc_checker', title, content, id, status)
+ yield "".join(make_collapsible_html('cc_checker', title, content, id, status))
inEntry = False
content = ""
@@ -281,7 +280,7 @@ def print_log_cc_checker(input):
# not a new entry, so part of the current entry's output
content += "%s\n" % line
- output += content
+ yield content
# This function does approximately the same as the following, following
# commented-out regular expression except that the regex doesn't quite
@@ -292,7 +291,6 @@ def print_log_cc_checker(input):
# (.*?)
# \n{3,}
# }{make_collapsible_html('cc_checker', "$1 $4", $5, $3, $2)}exgs
- return output
def make_collapsible_html(type, title, output, id, status=""):
@@ -311,17 +309,15 @@ def make_collapsible_html(type, title, output, id, status=""):
# note that we may be inside a <pre>, so we don't put any extra whitespace
# in this html
- ret = "<div class='%s unit %s' id='%s-%s'>" % (type, status, type, id)
- ret += "<a href=\"javascript:handle('%s');\">" % id
- ret += "<img id='img-%s' name='img-%s' alt='%s' src='%s' />" %(id, id, status, icon)
- ret += "<div class='%s title'>%s</div></a>" % (type, title)
- #ret += " "
- ret += "<div class='%s status %s'>%s</div>" % (type, status, status)
- ret += "<div class='%s output' id='output-%s'>" % (type, id)
+ yield "<div class='%s unit %s' id='%s-%s'>" % (type, status, type, id)
+ yield "<a href=\"javascript:handle('%s');\">" % id
+ yield "<img id='img-%s' name='img-%s' alt='%s' src='%s' />" %(id, id, status, icon)
+ yield "<div class='%s title'>%s</div></a>\n" % (type, title)
+ yield "<div class='%s status %s'>%s</div>\n" % (type, status, status)
+ yield "<div class='%s output' id='output-%s'>\n" % (type, id)
if output and len(output):
- ret += "<pre>%s</pre>>" % (output)
- ret += "</div></div>"
- return ret
+ yield "<pre>%s</pre>\n" % (output)
+ yield "</div></div>\n"
def diff_pretty(diff):
@@ -332,10 +328,8 @@ def diff_pretty(diff):
def web_paths(t, paths):
"""change the given source paths into links"""
if t.scm == "git":
- ret = ""
for path in paths:
- ret += " <a href=\"%s/?p=%s;a=history;f=%s%s;h=%s;hb=%s\">%s</a>" % (GITWEB_BASE, t.repo, t.subdir, path, t.branch, t.branch, path)
- return ret
+ yield " <a href=\"%s/?p=%s;a=history;f=%s%s;h=%s;hb=%s\">%s</a>" % (GITWEB_BASE, t.repo, t.subdir, path, t.branch, t.branch, path)
else:
raise Exception("Unknown scm %s" % t.scm)
@@ -370,25 +364,26 @@ def history_row_html(myself, entry, tree, changes):
</div>
<div class=\"author\">
<span class=\"label\">Author: </span>%s
- </div>""" % (myself, tree.name, entry.date, revision_url,
- myself, tree.name, entry.date, revision_url,
- msg, entry.author)
+ </div>""" % (
+ myself, tree.name, entry.date, revision_url,
+ myself, tree.name, entry.date, revision_url,
+ msg, entry.author)
(added, modified, removed) = changes
if modified:
yield "<div class=\"files\"><span class=\"label\">Modified: </span>"
- yield web_paths(tree, modified)
+ yield "".join(web_paths(tree, modified))
yield "</div>\n"
if added:
yield "<div class=\"files\"><span class=\"label\">Added: </span>"
- yield web_paths(tree, added)
+ yield "".join(web_paths(tree, added))
yield "</div>\n"
if removed:
yield "<div class=\"files\"><span class=\"label\">Removed: </span>"
- yield web_paths(tree, removed)
+ yield "".join(web_paths(tree, removed))
yield "</div>\n"
yield "</div>\n"
@@ -467,9 +462,6 @@ class ViewBuildPage(BuildFarmPage):
def render(self, myself, tree, host, compiler, rev, plain_logs=False):
"""view one build in detail"""
- uname = ""
- cflags = ""
- config = ""
build = self.buildfarm.get_build(tree, host, compiler, rev)
try:
(revision, revision_time) = build.revision_details()
@@ -492,31 +484,35 @@ class ViewBuildPage(BuildFarmPage):
finally:
f.close()
- if log:
- log = cgi.escape(log)
+ log = cgi.escape(log)
- m = re.search("(.*)", log)
- if m:
- uname = m.group(1)
- m = re.search("CFLAGS=(.*)", log)
- if m:
- cflags = m.group(1)
- m = re.search("configure options: (.*)", log)
- if m:
- config = m.group(1)
+ m = re.search("(.*)", log)
+ if m:
+ uname = m.group(1)
+ else:
+ uname = ""
+ m = re.search("CFLAGS=(.*)", log)
+ if m:
+ cflags = m.group(1)
+ else:
+ cflags = ""
+ m = re.search("configure options: (.*)", log)
+ if m:
+ config = m.group(1)
+ else:
+ config = ""
+ err = cgi.escape(err)
- if err:
- err = cgi.escape(err)
- yield '<h2>Host information:</h2>'
+ yield '<h2>Host information:</h2>\n'
host_web_file = "../web/%s.html" % host
if os.path.exists(host_web_file):
yield util.FileLoad(host_web_file)
yield "<table class='real'>\n"
- yield "<tr><td>Host:</td><td><a href='%s?function=View+Host;host=%s;tree=%s;"\
- "compiler=%s#'>%s</a> - %s</td></tr>\n" %\
- (myself, host, tree, compiler, host, self.buildfarm.hostdb.host(host).platform.encode("utf-8"))
+ yield ("<tr><td>Host:</td><td><a href='%s?function=View+Host;host=%s;tree=%s;"
+ "compiler=%s#'>%s</a> - %s</td></tr>\n" %
+ (myself, host, tree, compiler, host, self.buildfarm.hostdb.host(host).platform.encode("utf-8")))
yield "<tr><td>Uname:</td><td>%s</td></tr>\n" % uname
yield "<tr><td>Tree:</td><td>%s</td></tr>\n" % self.tree_link(myself, tree)
yield "<tr><td>Build Revision:</td><td>%s</td></tr>\n" % revision_link(myself, revision, tree)
@@ -534,46 +530,46 @@ class ViewBuildPage(BuildFarmPage):
if rev:
rev_var = ";revision=%s" % rev
- yield "<div id='log'>"
+ yield "<div id='log'>\n"
if not plain_logs:
- yield "<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s"\
- ";compiler=%s%s;plain=true' title='Switch to bland, non-javascript,"\
- " unstyled view'>Plain View</a></p>" % (myself, host, tree, compiler, rev_var)
+ yield ("<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s"
+ ";compiler=%s%s;plain=true' title='Switch to bland, non-javascript,"
+ " unstyled view'>Plain View</a></p>" % (myself, host, tree, compiler, rev_var))
- yield "<div id='actionList'>"
+ yield "<div id='actionList'>\n"
# These can be pretty wide -- perhaps we need to
# allow them to wrap in some way?
if err == "":
yield "<h2>No error log available</h2>\n"
else:
- yield "<h2>Error log:</h2>"
- yield make_collapsible_html('action', "Error Output", "\n%s" % err, "stderr-0", "errorlog")
+ yield "<h2>Error log:</h2>\n"
+ yield "".join(make_collapsible_html('action', "Error Output", "\n%s" % err, "stderr-0", "errorlog"))
if log == "":
- yield "<h2>No build log available</h2>"
+ yield "<h2>No build log available</h2>\n"
else:
yield "<h2>Build log:</h2>\n"
yield print_log_pretty(log)
- yield "<p><small>Some of the above icons derived from the <a href='http://www.gnome.org'>Gnome Project</a>'s stock icons.</small></p>"
- yield "</div>"
+ yield "<p><small>Some of the above icons derived from the <a href='http://www.gnome.org'>Gnome Project</a>'s stock icons.</small></p>\n"
+ yield "</div>\n"
else:
- yield "<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s;"\
- "compiler=%s%s' title='Switch to colourful, javascript-enabled, styled"\
- " view'>Enhanced View</a></p>" % (myself, host, tree, compiler, rev_var)
+ yield ("<p>Switch to the <a href='%s?function=View+Build;host=%s;tree=%s;"
+ "compiler=%s%s' title='Switch to colourful, javascript-enabled, styled"
+ " view'>Enhanced View</a></p>\n" % (myself, host, tree, compiler, rev_var))
if err == "":
- yield "<h2>No error log available</h2>"
+ yield "<h2>No error log available</h2>\n"
else:
yield '<h2>Error log:</h2>\n'
- yield '<div id="errorLog"><pre>%s</pre></div>' % err
+ yield '<div id="errorLog"><pre>%s</pre></div>\n' % err
if log == "":
- yield '<h2>No build log available</h2>'
+ yield '<h2>No build log available</h2>\n'
else:
yield '<h2>Build log:</h2>\n'
- yield '<div id="buildLog"><pre>%s</pre></div>' % log
+ yield '<div id="buildLog"><pre>%s</pre></div>\n' % log
- yield '</div>'
+ yield '</div>\n'
class ViewRecentBuildsPage(BuildFarmPage):
@@ -624,43 +620,43 @@ class ViewRecentBuildsPage(BuildFarmPage):
sorturl = "%s?tree=%s;function=Recent+Builds" % (myself, tree)
- yield "<div id='recent-builds' class='build-section'>"
- yield "<h2>Recent builds of %s (%s branch %s)</h2>" % (tree, t.scm, t.branch)
- yield "<table class='real'>"
- yield "<thead>"
- yield "<tr>"
- yield "<th><a href='%s;sortby=age' title='Sort by build age'>Age</a></th>" % sorturl
- yield "<th><a href='%s;sortby=revision' title='Sort by build revision'>Revision</a></th>" % sorturl
- yield "<th>Tree</th>"
- yield "<th><a href='%s;sortby=platform' title='Sort by platform'>Platform</a></th>" % sorturl
- yield "<th><a href='%s;sortby=host' title='Sort by host'>Host</a></th>" % sorturl
- yield "<th><a href='%s;sortby=compiler' title='Sort by compiler'>Compiler</a></th>" % sorturl
- yield "<th><a href='%s;sortby=status' title='Sort by status'>Status</a></th>" % sorturl
- yield "<tbody>"
+ yield "<div id='recent-builds' class='build-section'>\n"
+ yield "<h2>Recent builds of %s (%s branch %s)</h2>\n" % (tree, t.scm, t.branch)
+ yield "<table class='real'>\n"
+ yield "<thead>\n"
+ yield "<tr>\n"
--
build.samba.org
More information about the samba-cvs
mailing list