[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Sat Nov 13 05:08:33 MST 2010
The branch, master has been updated
via e4024b8 Various fixes found during ad-hoc testing.
via e771174 Add convenience function for avoiding database. Move database code to separate module.
via d4df7e4 Fix some formatting.
via ff7cecb More fixes for import-and-analyse.
via 1ca3392 Use named tuples, some more fixes for hostdb.
via ae0821a Warn early when creating host that already exists, move some storm code around.
via 99f99ba Move some functionality to Host.
via b26943a More storm usage.
via 549ef86 Use storm in host database.
via af937d5 Use storm to find builds.
via 75f1b64 Remove unecessary argument from Build().
via 75e2697 use storm.
from 2a37ec3 Be verbose when there's no previous build.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit e4024b82466d4b4966a4ac21ec0884cc296bf984
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 13:08:07 2010 +0100
Various fixes found during ad-hoc testing.
commit e77117456e12043754d1da19e19880d1c5899722
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 06:40:18 2010 +0100
Add convenience function for avoiding database. Move database code to separate module.
commit d4df7e442cb6acd8a4aacd8135a1b38c159925bc
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 06:05:10 2010 +0100
Fix some formatting.
commit ff7cecb4b0af868781a1a84563bc78921a250588
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 05:49:39 2010 +0100
More fixes for import-and-analyse.
commit 1ca339299de89b486979c73222e9faa9ead01b7e
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 05:24:47 2010 +0100
Use named tuples, some more fixes for hostdb.
commit ae0821a89cfc6810c4410f9f1db26d629c7ee41d
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 05:11:05 2010 +0100
Warn early when creating host that already exists, move some storm code around.
commit 99f99bab13945b6d6cdd499ecfdae845e08d8ad1
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 04:43:47 2010 +0100
Move some functionality to Host.
commit b26943a223f8215cf132c0176a0f1385d66d1f7c
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 04:16:47 2010 +0100
More storm usage.
commit 549ef86335847197cc58ddca5dc4d407f925814f
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 04:09:04 2010 +0100
Use storm in host database.
commit af937d5dc9d2c68aa0b69f7247e59ad0320ec570
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 03:50:04 2010 +0100
Use storm to find builds.
commit 75f1b64035ab24de2dcd38e5d0d914cb992cb2b7
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 03:09:46 2010 +0100
Remove unecessary argument from Build().
commit 75e2697b4116421545c64780fc4610c636461891
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 02:55:39 2010 +0100
use storm.
-----------------------------------------------------------------------
Summary of changes:
admin.py | 46 +++++---
buildfarm/__init__.py | 36 +-----
buildfarm/data.py | 128 ++++++++--------------
buildfarm/hostdb.py | 117 +++++++++++---------
buildfarm/sqldb.py | 217 +++++++++++++++++++++++++++++++++++++
buildfarm/tests/__init__.py | 19 +++-
buildfarm/tests/test_buildfarm.py | 9 +--
buildfarm/tests/test_data.py | 33 +------
buildfarm/tests/test_hostdb.py | 67 +++++-------
buildfarm/tests/test_sqldb.py | 78 +++++++++++++
import-and-analyse.py | 9 +-
mail-dead-hosts.py | 11 +-
web/build.py | 41 ++++----
13 files changed, 517 insertions(+), 294 deletions(-)
create mode 100644 buildfarm/sqldb.py
create mode 100644 buildfarm/tests/test_sqldb.py
Changeset truncated at 500 lines:
diff --git a/admin.py b/admin.py
index f17b26e..79c6ce7 100755
--- a/admin.py
+++ b/admin.py
@@ -70,16 +70,21 @@ else:
if op == "remove":
hostname = raw_input("Please enter hostname to delete: ")
try:
- db.deletehost(hostname)
+ db.deletehost(hostname.decode("utf-8"))
except hostdb.NoSuchHost, e:
print "No such host '%s'" % e.name
sys.exit(1)
else:
+ db.commit()
update_rsyncd_secrets()
update_hosts_list()
elif op == "modify":
hostname = raw_input("Please enter hostname to modify: ")
- host = db.host(hostname)
+ try:
+ host = db.host(hostname.decode("utf-8"))
+ except hostdb.NoSuchHost, e:
+ print "No such host '%s'" % e.name
+ sys.exit(1)
print "Owner: %s <%s>" % host.owner
print "Platform: %s" % host.platform
print ""
@@ -88,19 +93,13 @@ elif op == "modify":
mod_op = "platform"
if mod_op == "platform":
platform = raw_input("Enter new platform: ")
- try:
- db.update_platform(hostname, platform)
- except hostdb.NoSuchHost, e:
- print "No such host: %s" % e.name
- sys.exit(1)
+ host.update_platform(platform.decode("utf-8"))
+ db.commit()
elif mod_op == "owner":
owner = raw_input("Enter new owner's name: ")
owner_email = raw_input("Enter new owner's e-mail address: ")
- try:
- db.update_owner(hostname, owner, owner_email)
- except hostdb.NoSuchHost, e:
- print "No such host: %s" % e.name
- sys.exit(1)
+ host.update_owner(owner.decode("utf-8"), owner_email.decode("utf-8"))
+ db.commit()
else:
print "Unknown subcommand %s" % mod_op
sys.exit(1)
@@ -108,6 +107,13 @@ elif op == "modify":
update_hosts_list()
elif op == "add":
hostname = raw_input("Machine hostname: ")
+ try:
+ db.host(hostname.decode("utf-8"))
+ except hostdb.NoSuchHost, e:
+ pass
+ else:
+ print "A host with the name %s already exists." % e.name
+ sys.exit(1)
platform = raw_input("Machine platform (eg Fedora 9 x86_64): ")
owner = raw_input("Machine Owner Name: ")
owner_email = raw_input("Machine Owner E-mail: ")
@@ -123,10 +129,15 @@ elif op == "add":
line = raw_input("")
try:
- db.createhost(hostname, platform, owner, owner_email, password, "".join(permission))
+ db.createhost(hostname.decode("utf-8"), platform.decode("utf-8"),
+ owner.decode("utf-8"), owner_email.decode("utf-8"),
+ password.decode("utf-8"),
+ "".join(permission).decode("utf-8", "replace"))
except hostdb.HostAlreadyExists, e:
print "A host with the name %s already exists." % e.name
sys.exit(1)
+ else:
+ db.commit()
body = """
Welcome to the Samba.org build farm.
@@ -180,7 +191,11 @@ Thanks, your friendly Samba build farm administrator <build at samba.org>""" % owne
update_hosts_list()
elif op == "info":
hostname = raw_input("Hostname: ")
- host = db.host(hostname)
+ try:
+ host = db.host(hostname.decode("utf-8"))
+ except hostdb.NoSuchHost, e:
+ print "No such host '%s'" % e.name
+ sys.exit(1)
if host.fqdn:
opt_fqdn = " (%s)" % host.fqdn
else:
@@ -188,9 +203,6 @@ elif op == "info":
print "Host: %s%s" % (host.name, opt_fqdn)
print "Platform: %s" % host.platform
print "Owner: %s <%s>" % host.owner
-
- # Don't run the update of the text files
- sys.exit(0)
elif op == "list":
for host in db.host_ages():
if host.last_update:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index 6b44309..ba87fb1 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -20,7 +20,6 @@
import ConfigParser
import os
import re
-import sqlite3
class Tree(object):
@@ -99,13 +98,15 @@ class BuildFarm(object):
def _open_hostdb(self):
from buildfarm import hostdb
- return hostdb.HostDatabase(
- os.path.join(self.path, "hostdb.sqlite"))
+ return hostdb.PlainTextHostDatabase.from_file(os.path.join(self.webdir, "hosts.list"))
def _load_compilers(self):
from buildfarm import util
return set(util.load_list(os.path.join(self.webdir, "compilers.list")))
+ def commit(self):
+ pass
+
def lcov_status(self, tree):
"""get status of build"""
from buildfarm import data, util
@@ -130,9 +131,9 @@ class BuildFarm(object):
return self.upload_builds.get_build(tree, host, compiler)
def get_new_builds(self):
- hosts = set([host.name for host in self.hostdb.hosts()])
+ hostnames = set([host.name for host in self.hostdb.hosts()])
for build in self.upload_builds.get_new_builds():
- if build.tree in self.trees and build.compiler in self.compilers and build.host in hosts:
+ if build.tree in self.trees and build.compiler in self.compilers and build.host in hostnames:
yield build
@@ -188,29 +189,4 @@ class CachingBuildFarm(BuildFarm):
return perc
-class SQLCachingBuildFarm(BuildFarm):
-
- def __init__(self, path=None, db=None):
- self.db = db
- super(SQLCachingBuildFarm, self).__init__(path)
-
- def _get_db(self):
- if self.db is not None:
- return self.db
- else:
- return sqlite3.connect(os.path.join(self.path, "hostdb.sqlite"))
- def _open_build_results(self):
- from buildfarm import data
- return data.SQLCachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
- self.db)
-
-
-def setup_db(db):
- db.executescript("""
- CREATE TABLE IF NOT EXISTS host (name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int);
- CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);
- CREATE TABLE IF NOT EXISTS build (id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text);
- CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);
- CREATE TABLE IF NOT EXISTS test_run (build int, test text, result text, output text);
- """)
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 5518ce4..8629842 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -21,12 +21,11 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-from buildfarm import setup_db
from cStringIO import StringIO
+import collections
import hashlib
import os
import re
-import sqlite3
import time
import util
@@ -41,6 +40,9 @@ class BuildSummary(object):
self.status = status
+BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
+
+
class BuildStatus(object):
def __init__(self, stages=None, other_failures=None):
@@ -64,7 +66,7 @@ class BuildStatus(object):
return False
def _status_tuple(self):
- return [v for (k, v) in self.stages]
+ return [sr.result for sr in self.stages]
def regressed_since(self, other):
"""Check if this build has regressed since another build."""
@@ -91,7 +93,7 @@ class BuildStatus(object):
return cmp(other.stages, self.stages)
def __repr__(self):
- return "%s(%r)" % (self.__class__.__name__, (self.stages, self.other_failures))
+ return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
def check_dir_exists(kind, path):
@@ -111,16 +113,16 @@ def build_status_from_logs(log, err):
for l in log:
m = re.match("^([A-Z_]+) STATUS:(\s*\d+)$", l)
if m:
- stages.append((m.group(1), int(m.group(2).strip())))
+ stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
if m.group(1) == "TEST":
test_seen = 1
continue
m = re.match("^ACTION (PASSED|FAILED):\s+test$", l)
if m and not test_seen:
if m.group(1) == "PASSED":
- stages.append(("TEST", 0))
+ stages.append(BuildStageResult("TEST", 0))
else:
- stages.append(("TEST", 1))
+ stages.append(BuildStageResult("TEST", 1))
continue
if l.startswith("No space left on device"):
@@ -145,19 +147,19 @@ def build_status_from_logs(log, err):
if "No space left on device" in l:
ret.other_failures.add("disk full")
- def map_stage(name, result):
- if name != "TEST":
- return (name, result)
+ def map_stage(sr):
+ if sr.name != "TEST":
+ return sr
# TEST is special
if test_successes + test_failures == 0:
# No granular test output
- return ("TEST", result)
- if result == 1 and test_failures == 0:
+ return BuildStageResult("TEST", sr.result)
+ if sr.result == 1 and test_failures == 0:
ret.other_failures.add("inconsistent test result")
- return ("TEST", -1)
- return ("TEST", test_failures)
+ return BuildStageResult("TEST", -1)
+ return BuildStageResult("TEST", test_failures)
- ret.stages = [map_stage(name, result) for (name, result) in stages]
+ ret.stages = map(map_stage, stages)
return ret
@@ -175,25 +177,27 @@ class Build(object):
"""A single build of a tree on a particular host using a particular compiler.
"""
- def __init__(self, store, basename, tree, host, compiler, rev=None):
- self._store = store
+ def __init__(self, basename, tree, host, compiler, rev=None):
self.basename = basename
self.tree = tree
self.host = host
self.compiler = compiler
- self.revision = rev
+ self.commit_revision = self.revision = rev
def __repr__(self):
- if self.revision:
+ if self.revision is not None:
return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
else:
return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
- def remove(self):
+ def remove_logs(self):
os.unlink(self.basename + ".log")
if os.path.exists(self.basename+".err"):
os.unlink(self.basename+".err")
+ def remove(self):
+ self.remove_logs()
+
###################
# the mtime age is used to determine if builds are still happening
# on a host.
@@ -277,15 +281,19 @@ class CachingBuild(Build):
"""Build subclass that caches some of the results that are expensive
to calculate."""
- def revision_details(self):
+ def __init__(self, store, *args, **kwargs):
+ self._store = store
+ super(CachingBuild, self).__init__(*args, **kwargs)
if self.revision:
- cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
+ self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
else:
- cachef = self._store.cache_fname(self.tree, self.host, self.compiler)
+ self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler)
+
+ def revision_details(self):
st1 = os.stat("%s.log" % self.basename)
try:
- st2 = os.stat("%s.revision" % cachef)
+ st2 = os.stat("%s.revision" % self.cache_basename)
except OSError:
# File does not exist
st2 = None
@@ -293,7 +301,7 @@ class CachingBuild(Build):
# the ctime/mtime asymmetry is needed so we don't get fooled by
# the mtime update from rsync
if st2 and st1.st_ctime <= st2.st_mtime:
- (revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2)
+ (revid, timestamp) = util.FileLoad("%s.revision" % self.cache_basename).split(":", 2)
if timestamp == "":
timestamp = None
if revid == "":
@@ -301,34 +309,30 @@ class CachingBuild(Build):
return (revid, timestamp)
(revid, timestamp) = super(CachingBuild, self).revision_details()
if not self._store.readonly:
- util.FileSave("%s.revision" % cachef, "%s:%s" % (revid, timestamp or ""))
+ util.FileSave("%s.revision" % self.cache_basename, "%s:%s" % (revid, timestamp or ""))
return (revid, timestamp)
def err_count(self):
- cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
st1 = os.stat("%s.err" % self.basename)
try:
- st2 = os.stat("%s.errcount" % cachef)
+ st2 = os.stat("%s.errcount" % self.cache_basename)
except OSError:
# File does not exist
st2 = None
if st2 and st1.st_ctime <= st2.st_mtime:
- return util.FileLoad("%s.errcount" % cachef)
+ return util.FileLoad("%s.errcount" % self.cache_basename)
ret = super(CachingBuild, self).err_count()
if not self._store.readonly:
- util.FileSave("%s.errcount" % cachef, str(ret))
+ util.FileSave("%s.errcount" % self.cache_basename, str(ret))
return ret
def status(self):
- if self.revsion:
- cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)+".status"
- else:
- cachefile = self._store.cache_fname(self.tree, self.host, self.compiler)+".status"
+ cachefile = self.cache_basename + ".status"
st1 = os.stat("%s.log" % self.basename)
@@ -385,7 +389,7 @@ class UploadBuildResultStore(object):
logf = "%s.log" % basename
if not os.path.exists(logf):
raise NoSuchBuildError(tree, host, compiler)
- return Build(self, basename, tree, host, compiler)
+ return Build(basename, tree, host, compiler)
class CachingUploadBuildResultStore(UploadBuildResultStore):
@@ -425,7 +429,7 @@ class BuildResultStore(object):
logf = "%s.log" % basename
if not os.path.exists(logf):
raise NoSuchBuildError(tree, host, compiler, rev)
- return Build(self, basename, tree, host, compiler, rev)
+ return Build(basename, tree, host, compiler, rev)
def build_fname(self, tree, host, compiler, rev):
"""get the name of the build file"""
@@ -443,15 +447,9 @@ class BuildResultStore(object):
# skip the current build
if stat.st_nlink == 2:
continue
- build = self.get_build(tree, host, compiler, rev)
- r = {
- "STATUS": build.status(),
- "REVISION": rev,
- "TIMESTAMP": build.age_ctime(),
- }
- ret.append(r)
+ ret.append(self.get_build(tree, host, compiler, rev))
- ret.sort(lambda a, b: cmp(a["TIMESTAMP"], b["TIMESTAMP"]))
+ ret.sort(lambda a, b: cmp(a.age_mtime(), b.age_mtime()))
return ret
@@ -462,6 +460,12 @@ class BuildResultStore(object):
raise Exception("Unable to find revision in %r log" % build)
new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
+ try:
+ existing_build = self.get_build(build.tree, build.host, build.compiler, rev)
+ except NoSuchBuildError:
+ pass
+ else:
+ existing_build.remove_logs()
os.link(build.basename+".log", new_basename+".log")
if os.path.exists(build.basename+".err"):
os.link(build.basename+".err", new_basename+".err")
@@ -492,39 +496,3 @@ class CachingBuildResultStore(BuildResultStore):
def cache_fname(self, tree, host, compiler, rev):
return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
-
-
-class SQLCachingBuildResultStore(BuildResultStore):
-
- def __init__(self, basedir, db=None):
- super(SQLCachingBuildResultStore, self).__init__(basedir)
-
- if db is None:
- db = sqlite3.connect(":memory:")
- setup_db(db)
-
- self.db = db
-
- def get_previous_revision(self, tree, host, compiler, revision):
- cursor = self.db.execute("SELECT id FROM build WHERE tree = ? AND host = ? AND compiler = ? AND commit_revision = ?", (tree, host, compiler, revision))
- row = cursor.fetchone()
- if row is None:
- raise NoSuchBuildError(tree, host, compiler, revision)
- dbid = row[0]
- cursor = self.db.execute("SELECT commit_revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND id < ? ORDER BY id DESC LIMIT 1", (tree, host, compiler, dbid))
- row = cursor.fetchone()
- if row is None:
- raise NoSuchBuildError(tree, host, compiler, revision)
- return row[0]
-
- def get_latest_revision(self, tree, host, compiler):
- cursor = self.db.execute("SELECT commit_revision FROM build WHERE tree = ? AND host = ? AND compiler = ? ORDER BY id DESC LIMIT 1", (tree, host, compiler))
- row = cursor.fetchone()
- if row is None:
- raise NoSuchBuildError(tree, host, compiler)
- return row[0]
-
- def upload_build(self, build):
- super(SQLCachingBuildResultStore, self).upload_build(build)
- rev, timestamp = build.revision_details()
- self.db.execute("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (build.tree, rev, rev, build.host, build.compiler, build.log_checksum(), timestamp, repr(build.status())))
diff --git a/buildfarm/hostdb.py b/buildfarm/hostdb.py
index 2474505..3d7029c 100644
--- a/buildfarm/hostdb.py
+++ b/buildfarm/hostdb.py
@@ -18,9 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from buildfarm import setup_db
-
-import sqlite3
import time
@@ -44,87 +41,70 @@ class Host(object):
"""A host in the buildfarm."""
def __init__(self, name, owner=None, owner_email=None, password=None, platform=None,
- ssh_access=False, last_update=None, fqdn=None):
+ ssh_access=False, last_update=None, fqdn=None, join_time=None, permission=None):
self.name = name
if owner:
self.owner = (owner, owner_email)
else:
self.owner = None
+ if join_time is None:
+ self.join_time = time.time()
+ else:
+ self.join_time = join_time
+ self.permission = permission
self.password = password
self.platform = platform
self.ssh_access = ssh_access
--
build.samba.org
More information about the samba-cvs
mailing list