[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Fri Nov 12 15:58:19 MST 2010
The branch, master has been updated
via 99b7f0c Simplify revision_details() call.
via 720b006 Add upload_build for sqlite.
via 081607d add initial code for sql backed build result data.
from fdfaa5a Scan for builds then verify, rather than probing.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 99b7f0ce3b11bddac4580c9c0faec01085bad04c
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Nov 12 23:12:53 2010 +0100
Simplify revision_details() call.
commit 720b006124fd809bf38ca37d44367d107778e813
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Nov 12 23:08:22 2010 +0100
Add upload_build for sqlite.
commit 081607dfd99532b299e19df82dfc36b543502ed8
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Nov 12 22:51:50 2010 +0100
add initial code for sql backed build result data.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/__init__.py | 10 ++++
buildfarm/data.py | 89 ++++++++++++++++++-------------------
buildfarm/hostdb.py | 9 +---
buildfarm/tests/test_buildfarm.py | 4 +-
buildfarm/tests/test_data.py | 19 ++++++--
web/build.py | 12 +----
6 files changed, 74 insertions(+), 69 deletions(-)
Changeset truncated at 500 lines:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index 40522f7..822dc1f 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -185,3 +185,13 @@ class CachingBuildFarm(BuildFarm):
if not self.readonly:
util.FileSave(cachefile, perc)
return perc
+
+
+def setup_db(db):
+ db.executescript("""
+ CREATE TABLE IF NOT EXISTS host (name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int);
+ CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);
+ CREATE TABLE IF NOT EXISTS build (id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text);
+ CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);
+ CREATE TABLE IF NOT EXISTS test_run (build int, test text, result text, output text);
+ """)
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 0eaae74..888aa73 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -21,22 +21,23 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
+from buildfarm import setup_db
from cStringIO import StringIO
import hashlib
import os
import re
+import sqlite3
import time
import util
class BuildSummary(object):
- def __init__(self, host, tree, compiler, rev, status):
+ def __init__(self, host, tree, compiler, revision, status):
self.host = host
self.tree = tree
self.compiler = compiler
- self.rev = rev
+ self.revision = revision
self.status = status
@@ -180,11 +181,11 @@ class Build(object):
self.tree = tree
self.host = host
self.compiler = compiler
- self.rev = rev
+ self.revision = rev
def __repr__(self):
- if self.rev:
- return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.rev, self.tree, self.host, self.compiler)
+ if self.revision:
+ return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
else:
return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
@@ -228,9 +229,7 @@ class Build(object):
f.close()
def summary(self):
- (revid, commit_revid, timestamp) = self.revision_details()
- if commit_revid:
- revid = commit_revid
+ (revid, timestamp) = self.revision_details()
status = self.status()
return BuildSummary(self.host, self.tree, self.compiler, revid, status)
@@ -239,23 +238,19 @@ class Build(object):
:return: Tuple with revision id and timestamp (if available)
"""
-
revid = None
- commit_revid = None
timestamp = None
f = self.read_log()
try:
for l in f:
if l.startswith("BUILD COMMIT REVISION: "):
- commit_revid = l.split(":", 1)[1].strip()
- elif l.startswith("BUILD REVISION: "):
revid = l.split(":", 1)[1].strip()
elif l.startswith("BUILD COMMIT TIME"):
timestamp = l.split(":", 1)[1].strip()
finally:
f.close()
- return (revid, commit_revid, timestamp)
+ return (revid, timestamp)
def status(self):
"""get status of build
@@ -283,8 +278,8 @@ class CachingBuild(Build):
to calculate."""
def revision_details(self):
- if self.rev:
- cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
+ if self.revision:
+ cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
else:
cachef = self._store.cache_fname(self.tree, self.host, self.compiler)
st1 = os.stat("%s.log" % self.basename)
@@ -298,21 +293,19 @@ class CachingBuild(Build):
# the ctime/mtime asymmetry is needed so we don't get fooled by
# the mtime update from rsync
if st2 and st1.st_ctime <= st2.st_mtime:
- (revid, commit_revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2)
+ (revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2)
if timestamp == "":
timestamp = None
if revid == "":
revid = None
- if commit_revid == "":
- commit_revid = None
- return (revid, commit_revid, timestamp)
- (revid, commit_revid, timestamp) = super(CachingBuild, self).revision_details()
+ return (revid, timestamp)
+ (revid, timestamp) = super(CachingBuild, self).revision_details()
if not self._store.readonly:
- util.FileSave("%s.revision" % cachef, "%s:%s:%s" % (revid, commit_revid or "", timestamp or ""))
- return (revid, commit_revid, timestamp)
+ util.FileSave("%s.revision" % cachef, "%s:%s" % (revid, timestamp or ""))
+ return (revid, timestamp)
def err_count(self):
- cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
+ cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
st1 = os.stat("%s.err" % self.basename)
try:
@@ -332,8 +325,8 @@ class CachingBuild(Build):
return ret
def status(self):
- if self.rev:
- cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)+".status"
+ if self.revsion:
+ cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)+".status"
else:
cachefile = self._store.cache_fname(self.tree, self.host, self.compiler)+".status"
@@ -463,10 +456,7 @@ class BuildResultStore(object):
return ret
def upload_build(self, build):
- (rev, commit_rev, rev_timestamp) = build.revision_details()
-
- if commit_rev is not None:
- rev = commit_rev
+ (rev, rev_timestamp) = build.revision_details()
if not rev:
raise Exception("Unable to find revision in %r log" % build)
@@ -476,25 +466,9 @@ class BuildResultStore(object):
if os.path.exists(build.basename+".err"):
os.link(build.basename+".err", new_basename+".err")
- # FIXME:
- # $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
- # $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
-
def get_previous_revision(self, tree, host, compiler, revision):
raise NoSuchBuildError(tree, host, compiler, revision)
-"""
- def get_previous_revision(self, tree, host, compiler, revision):
- # Look up the database to find the previous status
- $st = $dbh->prepare("SELECT status, revision, commit_revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND revision != ? AND commit_revision != ? ORDER BY id DESC LIMIT 1")
- $st->execute( $tree, $host, $compiler, $rev, $commit)
-
- while ( my @row = $st->fetchrow_array ) {
- $old_status_html = @row[0]
- $old_rev = @row[1]
- $old_commit = @row[2]
- """
-
class CachingBuildResultStore(BuildResultStore):
@@ -515,3 +489,26 @@ class CachingBuildResultStore(BuildResultStore):
def cache_fname(self, tree, host, compiler, rev):
return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
+
+
+class SQLCachingBuildResultStore(BuildResultStore):
+
+ def __init__(self, basedir, db=None):
+ super(SQLCachingBuildResultStore, self).__init__(basedir)
+
+ if db is None:
+ db = sqlite3.connect(":memory:")
+ setup_db(db)
+
+ self.db = db
+
+ def get_previous_revision(self, tree, host, compiler, revision):
+ cursor = self.db.execute("SELECT revision FROM build WHERE tree = ? AND host = ? AND compiler = ? AND revision < ? ORDER BY id DESC LIMIT 1", (tree, host, compiler, revision))
+ row = cursor.fetchone()
+ if row is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return row[0]
+
+ def upload_build(self, build):
+ super(SQLCachingBuildResultStore, self).upload_build(build)
+ self.db.execute("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (build.tree, build.revision, build.revision, build.host, build.compiler, build.log_checksum(), build.age_mtime(), repr(build.status())))
diff --git a/buildfarm/hostdb.py b/buildfarm/hostdb.py
index 3c42757..2474505 100644
--- a/buildfarm/hostdb.py
+++ b/buildfarm/hostdb.py
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
+from buildfarm import setup_db
import sqlite3
import time
@@ -68,13 +69,7 @@ class HostDatabase(object):
else:
self.db = sqlite3.connect(filename)
self.filename = filename
- self.db.executescript("""
- CREATE TABLE IF NOT EXISTS host ( name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int );
- CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);
- CREATE TABLE IF NOT EXISTS build ( id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text);
- CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);
- CREATE TABLE IF NOT EXISTS test_run ( build int, test text, result text, output text);
- """)
+ setup_db(self.db)
self.db.commit()
def createhost(self, name, platform=None, owner=None, owner_email=None, password=None, permission=None):
diff --git a/buildfarm/tests/test_buildfarm.py b/buildfarm/tests/test_buildfarm.py
index 86e249a..02eda60 100644
--- a/buildfarm/tests/test_buildfarm.py
+++ b/buildfarm/tests/test_buildfarm.py
@@ -100,7 +100,7 @@ class BuildFarmTestBase(object):
self.assertEquals("tdb", build.tree)
self.assertEquals("charis", build.host)
self.assertEquals("cc", build.compiler)
- self.assertEquals("12", build.rev)
+ self.assertEquals("12", build.revision)
def test_get_build_no_rev(self):
path = self.create_mock_logfile("tdb", "charis", "cc",
@@ -109,7 +109,7 @@ class BuildFarmTestBase(object):
self.assertEquals("tdb", build.tree)
self.assertEquals("charis", build.host)
self.assertEquals("cc", build.compiler)
- self.assertIs(None, build.rev)
+ self.assertIs(None, build.revision)
class BuildFarmTests(BuildFarmTestBase, BuildFarmTestCase):
diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py
index 9824c51..4b60ad8 100755
--- a/buildfarm/tests/test_data.py
+++ b/buildfarm/tests/test_data.py
@@ -94,11 +94,10 @@ class BuildResultStoreTestBase(object):
self.create_mock_logfile("tdb", "charis", "cc", "12", contents="""
BUILD COMMIT REVISION: 43
bla
-BUILD REVISION: 42
BUILD COMMIT TIME: 3 August 2010
""")
build = self.x.get_build("tdb", "charis", "cc", "12")
- self.assertEquals(("42", "43", "3 August 2010"), build.revision_details())
+ self.assertEquals(("43", "3 August 2010"), build.revision_details())
def test_revision_details_no_timestamp(self):
self.create_mock_logfile("tdb", "charis", "cc", rev="12", contents="""
@@ -107,7 +106,7 @@ BUILD REVISION: 42
BLA
""")
build = self.x.get_build("tdb", "charis", "cc", "12")
- self.assertEquals(("42", "43", None), build.revision_details())
+ self.assertEquals(("43", None), build.revision_details())
def test_err_count(self):
self.create_mock_logfile("tdb", "charis", "cc", "12")
@@ -132,6 +131,9 @@ BUILD COMMIT REVISION: myrev
build = data.Build(None, path[:-4], "tdb", "charis", "cc")
self.assertRaises(Exception, self.x.upload_build, build)
+ def test_get_previous_revision(self):
+ self.assertRaises(data.NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12")
+
class BuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
@@ -158,6 +160,15 @@ class CachingBuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
"%s/cache/build.mytree.myhost.cc-123" % self.path)
+class SQLCachingBuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
+
+ def setUp(self):
+ super(SQLCachingBuildResultStoreTests, self).setUp()
+
+ self.x = data.SQLCachingBuildResultStore(
+ os.path.join(self.path, "data", "oldrevs"))
+
+
class BuildStatusFromLogs(testtools.TestCase):
def parse_logs(self, log, err):
@@ -330,5 +341,3 @@ class CachingUploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFar
self.assertEquals(
self.x.cache_fname("mytree", "myhost", "cc"),
"%s/cache/build.mytree.myhost.cc" % self.path)
-
-
diff --git a/web/build.py b/web/build.py
index 13700c1..43401de 100755
--- a/web/build.py
+++ b/web/build.py
@@ -280,9 +280,7 @@ def view_recent_builds(myself, tree, sort_by):
else:
age_mtime = build.age_mtime()
age_ctime = build.age_ctime()
- (revision, commit_revision, revision_time) = build.revision_details()
- if commit_revision:
- revision = commit_revision
+ (revision, revision_time) = build.revision_details()
if revision:
all_builds.append([age_ctime,
host.platform.encode("utf-8"),
@@ -393,9 +391,7 @@ def view_build(myself, tree, host, compiler, rev, plain_logs=False):
config = ""
build = buildfarm.get_build(tree, host, compiler, rev)
age_mtime = build.age_mtime()
- (revision, commit_revision, revision_time) = build.revision_details()
- if commit_revision:
- revision = commit_revision
+ (revision, revision_time) = build.revision_details()
status = build_status_html(myself, build)
if rev:
@@ -524,9 +520,7 @@ def view_host(myself, output_type, *requested_hosts):
except data.NoSuchBuildError:
pass
else:
- (revision, commit_revision, revision_time) = build.revision_details()
- if commit_revision:
- revision = commit_revision
+ (revision, revision_time) = build.revision_details()
age_mtime = build.age_mtime()
age_ctime = build.age_ctime()
warnings = build.err_count()
--
build.samba.org
More information about the samba-cvs
mailing list