[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Wed Nov 10 03:23:13 MST 2010
The branch, master has been updated
via 211abe8 More improvements to import-and-analyse.
from ee296a4 Simplify some code.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 211abe8c5db88a77017077495fd37f3a7d7cf019
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Wed Nov 10 11:23:57 2010 +0100
More improvements to import-and-analyse.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/__init__.py | 4 +-
buildfarm/data.py | 96 ++++++++++++++-----------------------------------
buildfarm/util.py | 3 +-
import-and-analyse.py | 18 ++++-----
4 files changed, 38 insertions(+), 83 deletions(-)
Changeset truncated at 500 lines:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index b12fb7b..2370940 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -71,7 +71,7 @@ class BuildFarm(object):
raise Exception("web directory %s does not exist" % self.webdir)
self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
self.builds = self._open_build_results()
- self.upload_builds = self._open_upload_builds_results()
+ self.upload_builds = self._open_upload_build_results()
self.hostdb = self._open_hostdb()
self.compilers = self._load_compilers()
self.lcovdir = os.path.join(self.path, "lcov/data")
@@ -131,7 +131,7 @@ class BuildFarm(object):
def get_new_builds(self):
from buildfarm import data
- for host in self.hostsdb.hosts():
+ for host in self.hostdb.hosts():
for tree in self.trees:
for compiler in self.compilers:
# By building the log file name this way, using only the list of
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 7062431..493badb 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -175,6 +175,10 @@ class Build(object):
self.host = host
self.compiler = compiler
self.rev = rev
+ if rev is None:
+ self.basename = self._store.build_fname(self.tree, self.host, self.compiler)
+ else:
+ self.basename = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
###################
# the mtime age is used to determine if builds are still happening
@@ -183,26 +187,22 @@ class Build(object):
def age_mtime(self):
"""get the age of build from mtime"""
- file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
-
- st = os.stat("%s.log" % file)
+ st = os.stat("%s.log" % self.basename)
return time.time() - st.st_mtime
def age_ctime(self):
"""get the age of build from ctime"""
- file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
-
- st = os.stat("%s.log" % file)
+ st = os.stat("%s.log" % self.basename)
return time.time() - st.st_ctime
def read_log(self):
"""read full log file"""
- return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".log", "r")
+ return open(self.basename+".log", "r")
def read_err(self):
"""read full err file"""
try:
- return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".err", 'r')
+ return open(self.basename+".err", 'r')
except IOError:
# No such file
return StringIO()
@@ -232,7 +232,7 @@ class Build(object):
timestamp = None
f = self.read_log()
try:
- for l in f.readlines():
+ for l in f:
if l.startswith("BUILD COMMIT REVISION: "):
commit_revid = l.split(":", 1)[1].strip()
elif l.startswith("BUILD REVISION: "):
@@ -270,9 +270,8 @@ class CachingBuild(Build):
to calculate."""
def revision_details(self):
- file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
- st1 = os.stat("%s.log" % file)
+ st1 = os.stat("%s.log" % self.basename)
try:
st2 = os.stat("%s.revision" % cachef)
@@ -297,9 +296,8 @@ class CachingBuild(Build):
return (revid, commit_revid, timestamp)
def err_count(self):
- file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
- st1 = os.stat("%s.err" % file)
+ st1 = os.stat("%s.err" % self.basename)
try:
st2 = os.stat("%s.errcount" % cachef)
@@ -318,10 +316,9 @@ class CachingBuild(Build):
return ret
def status(self):
- file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)+".status"
- st1 = os.stat("%s.log" % file)
+ st1 = os.stat("%s.log" % self.basename)
try:
st2 = os.stat(cachefile)
@@ -433,6 +430,21 @@ class BuildResultStore(object):
return ret
+ def upload_build(self, build):
+ (rev, commit_rev, rev_timestamp) = build.revision_details()
+
+ if commit_rev is not None:
+ rev = commit_rev
+
+ new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
+ os.rename(build.basename+".log", new_basename+".log")
+ if os.path.exists(build.basename+".err"):
+ os.rename(build.basename+".err", new_basename+".err")
+
+ # FIXME:
+ # $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+ # $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
+
"""
def get_previous_revision(self, tree, host, compiler, revision):
# Look up the database to find the previous status
@@ -443,60 +455,6 @@ class BuildResultStore(object):
$old_status_html = @row[0]
$old_rev = @row[1]
$old_commit = @row[2]
-
- def upload_build(self, build):
-
- my $expression = "SELECT checksum FROM build WHERE age >= ? AND tree = ? AND host = ? AND compiler = ?"
- my $st = $dbh->prepare($expression)
- $st->execute($stat->ctime, $tree, $host, $compiler)
- # Don't bother if we've already processed this file
- my $relevant_rows = $st->fetchall_arrayref()
- $st->finish()
-
- if relevant_rows > 0:
- return
-
- data = build.read_log()
- # Don't bother with empty logs, they have no meaning (and would all share the same checksum)
- if not data:
- return
-
- err = build.read_err()
- checksum = build.log_checksum()
- if ($dbh->selectrow_array("SELECT checksum FROM build WHERE checksum = '$checksum'")):
- $dbh->do("UPDATE BUILD SET age = ? WHERE checksum = ?", undef,
- ($stat->ctime, $checksum))
- continue
-
- (rev, rev_timestamp) = build.revision_details()
-
- status_html = db.build_status_from_logs(data, err)
-
- $st->finish()
-
- $st = $dbh->prepare("INSERT INTO build (tree, revision, commit_revision, host, compiler, checksum, age, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
- $st->execute($tree, $rev, $commit, $host, $compiler, $checksum, $stat->ctime, $status_html)
-
- $st->finish()
-
- cur_status = db.build_status_info_from_html(rev, commit, status_html)
-
- # If we were able to put this into the DB (ie, a
- # one-off event, so we won't repeat this), then also
- # hard-link the log files to the revision, if we know
- # it.
-
- # This ensures that the names under 'oldrev' are well known and well formed
- log_rev = self.build_fname(tree, host, compiler, commit) + ".log"
- err_rev = self.build_fname(tree, host, compiler, commit) + ".err"
- unlink $log_rev
- unlink $err_rev
- link($logfn . ".log", $log_rev) || die "Failed to link $logfn to $log_rev"
-
- # this prevents lots of links building up with err files
- copy($logfn . ".err", $err_rev) || die "Failed to copy $logfn to $err_rev"
- unlink($logfn . ".err")
- link($err_rev, $logfn . ".err")
"""
diff --git a/buildfarm/util.py b/buildfarm/util.py
index 60b263e..3b18164 100644
--- a/buildfarm/util.py
+++ b/buildfarm/util.py
@@ -67,6 +67,7 @@ def FileSave(filename, contents):
finally:
f.close()
+
def ChangeExtension(fname, ext):
"""return a filename with a changed extension"""
try:
@@ -83,8 +84,6 @@ def dhm_time(sec):
hour = int(sec / (60*60)) % 24;
min = int(sec / 60) % 60;
- ret = ""
-
if sec < 0:
return "-"
diff --git a/import-and-analyse.py b/import-and-analyse.py
index d45be19..de7f08d 100644
--- a/import-and-analyse.py
+++ b/import-and-analyse.py
@@ -12,7 +12,6 @@ on recent commits.
from buildfarm import (
BuildFarm,
- data,
hostdb,
)
import commands
@@ -23,8 +22,6 @@ import os
import re
import smtplib
-dry_run = True
-
parser = optparse.OptionParser("import-and-analyse [options]")
parser.add_option("--dry-run", help="Will cause the script to send output to stdout instead of to sendmail.", action="store_true")
parser.add_option("--verbose", help="Be verbose", action="count")
@@ -35,7 +32,7 @@ UNPACKED_DIR = "/home/ftp/pub/unpacked"
# we open readonly here as only apache(www-run) has write access
buildfarm = BuildFarm()
-db = data.BuildResultStore(os.path.abspath(os.path.dirname(__file__)), True)
+db = buildfarm.builds
hostsdb = buildfarm.hostdb
smtp = smtplib.SMTP()
@@ -105,18 +102,18 @@ def check_and_send_mails(tree, host, compiler, cur, old):
(old_rev, old_rev_timestamp) = old.revision_details()
old_status = old.status()
- if dry_run:
+ if opts.dry_run:
print "rev=%s status=%s" % (cur_rev, cur_status)
print "old rev=%s status=%s" % (old_rev, old_status)
if not cur_status.regressed_since(old_status):
- if dry_run:
+ if opts.dry_run:
print "the build didn't get worse since %r" % old_status
return
log = get_log(tree, cur, old)
if not log:
- if dry_run:
+ if opts.dry_run:
print "no log"
return
@@ -135,7 +132,8 @@ See http://build.samba.org/?function=View+Build;host=%(host)s;tree=%(tree)s;comp
The build may have been broken by one of the following commits:
%(change_log)s
- """ % {"tree": tree, "host": host, "compiler": compiler, "change_log": log.change_log, "scm": t.scm, "branch": t.branch}
+ """ % {"tree": tree, "host": host, "compiler": compiler, "change_log": log.change_log, "scm": t.scm, "branch": t.branch,
+ "cur_rev": cur_rev, "old_rev": old_rev, "cur_status": cur_status, "old_status": old_status }
msg = MIMEText(body)
msg["Subject"] = "BUILD of %s:%s BROKEN on %s with %s AT REVISION %s" % (tree, t.branch, host, compiler, cur_rev)
@@ -150,7 +148,7 @@ for build in buildfarm.get_new_builds():
db.upload_build(build)
- (rev, commit_rev, rev_timestamp) = db.revision_details()
+ (rev, commit_rev, rev_timestamp) = build.revision_details()
try:
prev_rev = db.get_previous_revision(build.tree, build.host, build.compiler, rev)
@@ -159,7 +157,7 @@ for build in buildfarm.get_new_builds():
continue
else:
prev_build = db.get_build(build.tree, build.host, build.compiler, prev_rev)
- check_and_send_mails(build.tree, build.host, build.compiler, build.status(), prev_build.status())
+ check_and_send_mails(build.tree, build.host, build.compiler, build, prev_build)
smtp.quit()
--
build.samba.org
More information about the samba-cvs
mailing list