[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Fri Dec 3 17:28:13 MST 2010
The branch, master has been updated
via a0c2c75 Merge StormCachingBuildFarm and BuildFarm.
via c169c3b Kill StormCachingBuildFarm - regular build farm now uses Storm.
via f708032 Kill 'plain' old revision database.
via 262c55c Fix a bunch of tests.
via 1922c25 Add testrepository configuration.
via de73273 Add test_suite function.
via 198a037 Use convenience function for finding duplicate builds.
via 048eb62 Remove the no longer used broken_repo script.
via d3b7257 Add get_by_checksum function.
via 60fd7c7 Simplify code a bit.
from bba2790 fix "test: =: unary operator expected" error as one of the vars could be empty?
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit a0c2c7539958dbdeed5c3dbc36f0f08ba47a09a9
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Dec 4 01:13:20 2010 +0100
Merge StormCachingBuildFarm and BuildFarm.
commit c169c3bb130981055d324fb138a4037f773b6d05
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Dec 4 00:00:03 2010 +0100
Kill StormCachingBuildFarm - regular build farm now uses Storm.
commit f708032e620fb4a2500ffc085449e4c699b184f1
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Dec 3 23:32:06 2010 +0100
Kill 'plain' old revision database.
commit 262c55c440aca77d36959fc97271a5ee3b816e1a
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Dec 3 23:04:36 2010 +0100
Fix a bunch of tests.
commit 1922c25adaea9708dee885dec635fdc4f9cd35df
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Dec 3 15:27:29 2010 +0100
Add testrepository configuration.
commit de73273f59d9d45114f5bdead553a6a99cc2e812
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Dec 3 15:25:20 2010 +0100
Add test_suite function.
commit 198a0378a43c966698b41f8e7a9ae1187d470f4b
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Fri Dec 3 14:28:29 2010 +0100
Use convenience function for finding duplicate builds.
commit 048eb62b7d28b18c202230e8bae9ace1d4770bca
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Dec 1 17:41:57 2010 +0100
Remove the no longer used broken_repo script.
commit d3b7257f162f8a87812aff251deb6ac4f17e1583
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Tue Nov 30 12:37:57 2010 +0100
Add get_by_checksum function.
commit 60fd7c7417608fd588c482505adc8b002c4bba5d
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Tue Nov 30 12:37:34 2010 +0100
Simplify code a bit.
-----------------------------------------------------------------------
Summary of changes:
.bzrignore | 1 +
.testr.conf | 3 +
admin.py | 4 +-
broken_report.sh | 158 -------------------------------
buildfarm/__init__.py | 84 +++++++----------
buildfarm/build.py | 151 ++++++++++++++++++++++++------
buildfarm/hostdb.py | 28 ------
buildfarm/sqldb.py | 185 +------------------------------------
buildfarm/tests/__init__.py | 33 +++++--
buildfarm/tests/test_build.py | 44 ++++++----
buildfarm/tests/test_buildfarm.py | 18 ++--
buildfarm/tests/test_sqldb.py | 41 +--------
buildfarm/web/__init__.py | 4 +-
builds.py | 4 +-
import-and-analyse.py | 7 +-
mail-dead-hosts.py | 4 +-
tools/fix.py | 4 +-
17 files changed, 242 insertions(+), 531 deletions(-)
create mode 100644 .testr.conf
delete mode 100755 broken_report.sh
Changeset truncated at 500 lines:
diff --git a/.bzrignore b/.bzrignore
index 23886d3..c5987b3 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -1,3 +1,4 @@
_trial_temp
db
data
+.testrepository
diff --git a/.testr.conf b/.testr.conf
new file mode 100644
index 0000000..8465f49
--- /dev/null
+++ b/.testr.conf
@@ -0,0 +1,3 @@
+[DEFAULT]
+test_command=PYTHONPATH=. python -m subunit.run $IDLIST
+test_id_list_default=buildfarm.tests.test_suite
diff --git a/admin.py b/admin.py
index 07c0164..60679f8 100755
--- a/admin.py
+++ b/admin.py
@@ -18,9 +18,9 @@
#
from buildfarm import (
+ BuildFarm,
hostdb,
)
-from buildfarm.sqldb import StormCachingBuildFarm
import commands
import os
import smtplib
@@ -28,7 +28,7 @@ import sys
import time
from email.MIMEText import MIMEText
-buildfarm = StormCachingBuildFarm()
+buildfarm = BuildFarm()
def update_rsyncd_secrets():
temp_rsyncd_secrets = os.path.join(os.path.dirname(__file__), "../rsyncd.secrets.new")
diff --git a/broken_report.sh b/broken_report.sh
deleted file mode 100755
index 05ea471..0000000
--- a/broken_report.sh
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/bin/bash
-# This shell script produces an email comparing current broken status
-# to that of the last run of this script
-#
-# Copyright (C) Vance Lankhaar <vance at samba.org> 2005
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-BASEDIR="/home/build/master"
-RESULT_CACHE="$BASEDIR/cache/broken_results.txt"
-RESULT_URL="http://build.samba.org/?function=Text_Summary"
-FULL_RESULT_URL="http://build.samba.org/"
-
-# split into two parts to obfuscate from spam engines
-RESULT_EMAIL_USER="samba-cvs"
-RESULT_EMAIL_DOMAIN="lists.samba.org"
-
-# End editable variables
-##################################################
-
-
-# exit immediately on failure
-set -e
-
-# fail on attempting to use unassigned variable
-set -u
-
-# clean up environment a little
-PATH="/usr/local/bin:/usr/bin:/bin"
-IFS="
-"
-
-
-##################################################
-# get the broken report from the build farm
-function get_results {
-
- rm -f "$RESULT_CACHE"
-
- wget --output-file="$RESULT_CACHE".log \
- --output-document="$RESULT_CACHE" \
- "$RESULT_URL"
-
- if [ $? -ne 0 ]; then
- echo "Could not retrieve results:" >&2
- cat "$RESULT_CACHE".log >&2
- return 1;
- else
- rm -rf "$RESULT_CACHE".log
- fi
-}
-
-##################################################
-# compare the results of build
-function compare_results {
-
- if [ ! -e "$RESULT_CACHE" ]; then
- echo "Could not locate currrent results. $RESULT_CACHE not found" >&2
- return 1
- fi
-
- if [ ! -e "$RESULT_CACHE".old ]; then
- echo "Could not locate old results. $RESULT_CACHE.old not found" >&2
- return 1
- fi
-
- diff -u "$RESULT_CACHE".old "$RESULT_CACHE" >> "$RESULT_CACHE".report || true
-
- return 0
-}
-
-##################################################
-# send report to the above-set email address
-function send_report {
-
- if [ ! -e "$RESULT_CACHE".report ]; then
- echo "Results ($RESULT_CACHE.report) not found" >&2
- return 1
- fi
-
- subject=$1
- if [ x"$subject" = x ]; then
- subject="Build Farm Status"
- fi
-
- mail -s "$subject" "$RESULT_EMAIL_USER"@"$RESULT_EMAIL_DOMAIN" < "$RESULT_CACHE".report
-}
-
-##################################################
-##################################################
-
-if [ $# -gt 0 ]; then
- if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
- echo "Usage: broken_report.sh"
- echo
- echo "This shell script produces an email comparing current broken"
- echo "status to that of the last run of this script."
- echo
- echo "The source and destination are configurable within the script."
- exit 1
- fi
-fi
-
-
-##################################################
-# prepares the report, including subject and "blurb"
-rm -f "$RESULT_CACHE".report "$RESULT_CACHE".old
-
-if [ -e "$RESULT_CACHE" ]; then
- mv -f "$RESULT_CACHE" "$RESULT_CACHE".old
-else
- # get results if they don't exist
- get_results
- exit $?
-fi
-
-get_results
-
-if [ $? -ne 0 ]; then
- echo "Failed to get the results. Bailing." >&2
-
- # remove any new results, as they're almost certainly foul
- rm -f "$RESULT_CACHE"
- mv -f "$RESULT_CACHE".old "$RESULT_CACHE"
-
- exit 1
-fi
-
-
-(
- # set report subject to show updated time
- echo "URL: $FULL_RESULT_URL"
- echo
-) > "$RESULT_CACHE".report
-
-compare_results
-
-if [ $? -ne 0 ]; then
- echo "Failed to compare results. Bailing." >&2
-
- exit 1
-fi
-
-subject=$(head -n 1 "$RESULT_CACHE")
-send_report "$subject"
-
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index b515b12..1413421 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -17,36 +17,16 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+from buildfarm.sqldb import distinct_builds, Cast, StormBuild, setup_schema, StormHostDatabase
+from buildfarm.tree import Tree
+from storm.database import create_database
+from storm.expr import Desc
+from storm.store import Store
+
import ConfigParser
import os
import re
-GIT_ROOT = "/data/git"
-
-
-class Tree(object):
- """A tree to build."""
-
- def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
- self.name = name
- self.repo = repo
- self.scm = scm
- self.branch = branch
- self.subdir = subdir
- self.srcdir = srcdir
- self.scm = scm
-
- def get_branch(self):
- if self.scm == "git":
- from buildfarm.history import GitBranch
- return GitBranch(os.path.join(GIT_ROOT, self.repo), self.branch)
- else:
- raise NotImplementedError(self.scm)
-
- def __repr__(self):
- return "<%s %r>" % (self.__class__.__name__, self.name)
-
-
def read_trees_from_conf(path):
"""Read trees from a configuration file.
@@ -76,7 +56,9 @@ class BuildFarm(object):
OLDAGE = 60*60*4,
DEADAGE = 60*60*24*4
- def __init__(self, path=None):
+ def __init__(self, path=None, store=None, timeout=0.5):
+ self.timeout = timeout
+ self.store = store
if path is None:
path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
self.path = path
@@ -94,9 +76,9 @@ class BuildFarm(object):
return "%s(%r)" % (self.__class__.__name__, self.path)
def _open_build_results(self):
- from buildfarm.build import BuildResultStore
path = os.path.join(self.path, "data", "oldrevs")
- return BuildResultStore(path)
+ from buildfarm.build import BuildResultStore
+ return BuildResultStore(path, self._get_store())
def _open_upload_build_results(self):
from buildfarm.build import UploadBuildResultStore
@@ -104,15 +86,15 @@ class BuildFarm(object):
return UploadBuildResultStore(path)
def _open_hostdb(self):
- from buildfarm import hostdb
- return hostdb.PlainTextHostDatabase.from_file(os.path.join(self.webdir, "hosts.list"))
+ return StormHostDatabase(self._get_store())
def _load_compilers(self):
from buildfarm import util
return set(util.load_list(os.path.join(self.webdir, "compilers.list")))
def commit(self):
- pass
+ if self.store is not None:
+ self.store.commit()
def lcov_status(self, tree):
"""get status of build"""
@@ -144,27 +126,31 @@ class BuildFarm(object):
yield build
def get_last_builds(self):
- return sorted(self.get_new_builds(), reverse=True)
+ result = self._get_store().find(StormBuild)
+ return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
def get_tree_builds(self, tree):
- ret = []
- for build in self.builds.get_all_builds():
- if build.tree == tree:
- ret.append(build)
- ret.sort(reverse=True)
- return ret
+ result = self._get_store().find(StormBuild,
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"))
+ return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
def host_last_build(self, host):
return max([build.upload_time for build in self.get_host_builds(host)])
def get_host_builds(self, host):
- from buildfarm.build import NoSuchBuildError
- ret = []
- for compiler in self.compilers:
- for tree in sorted(self.trees.keys()):
- try:
- ret.append(self.get_build(tree, host, compiler))
- except NoSuchBuildError:
- pass
- ret.sort(reverse=True)
- return ret
+ result = self._get_store().find(StormBuild, StormBuild.host == host)
+ return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
+
+ def _get_store(self):
+ if self.store is not None:
+ return self.store
+ db_path = os.path.join(self.path, "db", "hostdb.sqlite")
+ db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
+ self.store = Store(db)
+ setup_schema(self.store)
+ return self.store
+
+ def get_revision_builds(self, tree, revision=None):
+ return self._get_store().find(StormBuild,
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
diff --git a/buildfarm/build.py b/buildfarm/build.py
index b218788..0cbe3bf 100644
--- a/buildfarm/build.py
+++ b/buildfarm/build.py
@@ -26,6 +26,9 @@ import collections
import hashlib
import os
import re
+from storm.locals import Int, RawStr
+from storm.store import Store
+from storm.expr import Desc
import time
@@ -384,34 +387,74 @@ class UploadBuildResultStore(object):
return Build(basename, tree, host, compiler)
+class StormBuild(Build):
+ __storm_table__ = "build"
+
+ id = Int(primary=True)
+ tree = RawStr()
+ revision = RawStr()
+ host = RawStr()
+ compiler = RawStr()
+ checksum = RawStr()
+ upload_time = Int(name="age")
+ status_str = RawStr(name="status")
+ basename = RawStr()
+ host_id = Int()
+ tree_id = Int()
+ compiler_id = Int()
+
+ def status(self):
+ return BuildStatus.__deserialize__(self.status_str)
+
+ def revision_details(self):
+ return self.revision
+
+ def log_checksum(self):
+ return self.checksum
+
+ def remove(self):
+ super(StormBuild, self).remove()
+ Store.of(self).remove(self)
+
+ def remove_logs(self):
+ super(StormBuild, self).remove_logs()
+ self.basename = None
+
+
class BuildResultStore(object):
"""The build farm build result database."""
- def __init__(self, path):
- """Open the database.
+ def __init__(self, basedir, store=None):
+ from buildfarm.sqldb import memory_store
+ if store is None:
+ store = memory_store()
- :param path: Build result base directory
- """
- self.path = path
+ self.store = store
+ self.path = basedir
def __contains__(self, build):
try:
- if build.revision:
- rev = build.revision
- else:
- rev = build.revision_details()
- self.get_build(build.tree, build.host, build.compiler, rev)
+ self.get_by_checksum(build.log_checksum())
+ return True
except NoSuchBuildError:
return False
- else:
- return True
- def get_build(self, tree, host, compiler, rev, checksum=None):
- basename = self.build_fname(tree, host, compiler, rev)
- logf = "%s.log" % basename
- if not os.path.exists(logf):
- raise NoSuchBuildError(tree, host, compiler, rev)
- return Build(basename, tree, host, compiler, rev)
+ def get_build(self, tree, host, compiler, revision=None, checksum=None):
+ from buildfarm.sqldb import Cast
+ expr = [
+ Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
+ Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
+ Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
+ ]
+ if revision is not None:
+ expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
+ if checksum is not None:
+ expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
+ result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
+ ret = result.first()
+ if ret is None:
+ raise NoSuchBuildError(tree, host, compiler, revision)
+ return ret
def build_fname(self, tree, host, compiler, rev):
"""get the name of the build file"""
@@ -433,15 +476,24 @@ class BuildResultStore(object):
yield self.get_build(tree, host, compiler, rev)
def get_old_builds(self, tree, host, compiler):
- """get a list of old builds and their status."""
- ret = []
- for build in self.get_all_builds():
- if build.tree == tree and build.host == host and build.compiler == compiler:
- ret.append(build)
- ret.sort(lambda a, b: cmp(a.upload_time, b.upload_time))
- return ret
+ result = self.store.find(StormBuild,
+ StormBuild.tree == tree,
+ StormBuild.host == host,
+ StormBuild.compiler == compiler)
+ return result.order_by(Desc(StormBuild.upload_time))
def upload_build(self, build):
+ from buildfarm.sqldb import Cast, StormHost
+ try:
+ existing_build = self.get_by_checksum(build.log_checksum())
+ except NoSuchBuildError:
+ pass
+ else:
+ # Already present
+ assert build.tree == existing_build.tree
+ assert build.host == existing_build.host
+ assert build.compiler == existing_build.compiler
+ return existing_build
rev = build.revision_details()
new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
@@ -457,10 +509,53 @@ class BuildResultStore(object):
os.link(build.basename+".log", new_basename+".log")
if os.path.exists(build.basename+".err"):
os.link(build.basename+".err", new_basename+".err")
- return Build(new_basename, build.tree, build.host, build.compiler, rev)
+ new_basename = self.build_fname(build.tree, build.host, build.compiler,
+ rev)
+ new_build = StormBuild(new_basename, build.tree, build.host,
+ build.compiler, rev)
+ new_build.checksum = build.log_checksum()
+ new_build.upload_time = build.upload_time
+ new_build.status_str = build.status().__serialize__()
+ new_build.basename = new_basename
+ host = self.store.find(StormHost,
+ Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
+ assert host is not None, "Unable to find host %r" % build.host
+ new_build.host_id = host.id
--
build.samba.org
More information about the samba-cvs
mailing list