[SCM] build.samba.org - branch master updated
Jelmer Vernooij
jelmer at samba.org
Sat Nov 13 07:10:22 MST 2010
The branch, master has been updated
via be38b55 Limit history.
via 7f26933 Move file cache specific code to filecache module.
from 0ab262b attempt to speed up log parsing by compiling regexes beforehand.
http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit be38b55bbaac14dee4ad51bf7c6225932752f56e
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 15:10:17 2010 +0100
Limit history.
commit 7f26933e3dce416a697fb3ac5b2681d178afccda
Author: Jelmer Vernooij <jelmer at samba.org>
Date: Sat Nov 13 14:54:09 2010 +0100
Move file cache specific code to filecache module.
-----------------------------------------------------------------------
Summary of changes:
buildfarm/__init__.py | 55 ----------
buildfarm/data.py | 117 ---------------------
buildfarm/filecache.py | 200 +++++++++++++++++++++++++++++++++++++
buildfarm/history.py | 18 ++--
buildfarm/tests/test_buildfarm.py | 7 --
buildfarm/tests/test_data.py | 27 -----
buildfarm/tests/test_filecache.py | 70 +++++++++++++
web/build.py | 6 +-
8 files changed, 284 insertions(+), 216 deletions(-)
create mode 100644 buildfarm/filecache.py
create mode 100644 buildfarm/tests/test_filecache.py
Changeset truncated at 500 lines:
diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py
index 76a4146..58b66fb 100644
--- a/buildfarm/__init__.py
+++ b/buildfarm/__init__.py
@@ -140,58 +140,3 @@ class BuildFarm(object):
for build in self.upload_builds.get_new_builds():
if build.tree in self.trees and build.compiler in self.compilers and build.host in hostnames:
yield build
-
-
-class CachingBuildFarm(BuildFarm):
-
- def __init__(self, path=None, readonly=False, cachedirname=None):
- self._cachedirname = cachedirname
- self.readonly = readonly
- super(CachingBuildFarm, self).__init__(path)
-
- def _get_cachedir(self):
- if self._cachedirname is not None:
- return os.path.join(self.path, self._cachedirname)
- else:
- return os.path.join(self.path, "cache")
-
- def _open_build_results(self):
- from buildfarm import data
- return data.CachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
- self._get_cachedir(), readonly=self.readonly)
-
- def _open_upload_build_results(self):
- from buildfarm import data
- return data.CachingUploadBuildResultStore(os.path.join(self.path, "data", "upload"),
- self._get_cachedir(), readonly=self.readonly)
-
- def lcov_status(self, tree):
- """get status of build"""
- from buildfarm import data, util
- cachefile = os.path.join(self._get_cachedir(),
- "lcov.%s.%s.status" % (self.LCOVHOST, tree))
- file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
- try:
- st1 = os.stat(file)
- except OSError:
- # File does not exist
- raise data.NoSuchBuildError(tree, self.LCOVHOST, "lcov")
- try:
- st2 = os.stat(cachefile)
- except OSError:
- # file does not exist
- st2 = None
-
- if st2 and st1.st_ctime <= st2.st_mtime:
- ret = util.FileLoad(cachefile)
- if ret == "":
- return None
- return ret
-
- perc = super(CachingBuildFarm, self).lcov_status(tree)
- if not self.readonly:
- util.FileSave(cachefile, perc)
- return perc
-
-
-
diff --git a/buildfarm/data.py b/buildfarm/data.py
index 0bd1498..4120730 100644
--- a/buildfarm/data.py
+++ b/buildfarm/data.py
@@ -27,7 +27,6 @@ import hashlib
import os
import re
import time
-import util
class BuildSummary(object):
@@ -277,82 +276,6 @@ class Build(object):
return len(file.readlines())
-class CachingBuild(Build):
- """Build subclass that caches some of the results that are expensive
- to calculate."""
-
- def __init__(self, store, *args, **kwargs):
- self._store = store
- super(CachingBuild, self).__init__(*args, **kwargs)
- if self.revision:
- self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
- else:
- self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler)
-
- def revision_details(self):
- st1 = os.stat("%s.log" % self.basename)
-
- try:
- st2 = os.stat("%s.revision" % self.cache_basename)
- except OSError:
- # File does not exist
- st2 = None
-
- # the ctime/mtime asymmetry is needed so we don't get fooled by
- # the mtime update from rsync
- if st2 and st1.st_ctime <= st2.st_mtime:
- (revid, timestamp) = util.FileLoad("%s.revision" % self.cache_basename).split(":", 2)
- if timestamp == "":
- timestamp = None
- if revid == "":
- revid = None
- return (revid, timestamp)
- (revid, timestamp) = super(CachingBuild, self).revision_details()
- if not self._store.readonly:
- util.FileSave("%s.revision" % self.cache_basename, "%s:%s" % (revid, timestamp or ""))
- return (revid, timestamp)
-
- def err_count(self):
- st1 = os.stat("%s.err" % self.basename)
-
- try:
- st2 = os.stat("%s.errcount" % self.cache_basename)
- except OSError:
- # File does not exist
- st2 = None
-
- if st2 and st1.st_ctime <= st2.st_mtime:
- return util.FileLoad("%s.errcount" % self.cache_basename)
-
- ret = super(CachingBuild, self).err_count()
-
- if not self._store.readonly:
- util.FileSave("%s.errcount" % self.cache_basename, str(ret))
-
- return ret
-
- def status(self):
- cachefile = self.cache_basename + ".status"
-
- st1 = os.stat("%s.log" % self.basename)
-
- try:
- st2 = os.stat(cachefile)
- except OSError:
- # No such file
- st2 = None
-
- if st2 and st1.st_ctime <= st2.st_mtime:
- return eval(util.FileLoad(cachefile))
-
- ret = super(CachingBuild, self).status()
-
- if not self._store.readonly:
- util.FileSave(cachefile, repr(ret))
-
- return ret
-
-
class UploadBuildResultStore(object):
def __init__(self, path):
@@ -392,28 +315,6 @@ class UploadBuildResultStore(object):
return Build(basename, tree, host, compiler)
-class CachingUploadBuildResultStore(UploadBuildResultStore):
-
- def __init__(self, basedir, cachedir, readonly=False):
- """Open the database.
-
- :param readonly: Whether to avoid saving cache files
- """
- super(CachingUploadBuildResultStore, self).__init__(basedir)
- self.cachedir = cachedir
- self.readonly = readonly
-
- def cache_fname(self, tree, host, compiler):
- return os.path.join(self.cachedir, "build.%s.%s.%s" % (tree, host, compiler))
-
- def get_build(self, tree, host, compiler):
- basename = self.build_fname(tree, host, compiler)
- logf = "%s.log" % basename
- if not os.path.exists(logf):
- raise NoSuchBuildError(tree, host, compiler)
- return CachingBuild(self, basename, tree, host, compiler)
-
-
class BuildResultStore(object):
"""The build farm build result database."""
@@ -477,22 +378,4 @@ class BuildResultStore(object):
raise NoSuchBuildError(tree, host, compiler)
-class CachingBuildResultStore(BuildResultStore):
-
- def __init__(self, basedir, cachedir, readonly=False):
- super(CachingBuildResultStore, self).__init__(basedir)
-
- self.cachedir = cachedir
- check_dir_exists("cache", self.cachedir)
-
- self.readonly = readonly
-
- def get_build(self, tree, host, compiler, rev):
- basename = self.build_fname(tree, host, compiler, rev)
- logf = "%s.log" % basename
- if not os.path.exists(logf):
- raise NoSuchBuildError(tree, host, compiler, rev)
- return CachingBuild(self, basename, tree, host, compiler, rev)
- def cache_fname(self, tree, host, compiler, rev):
- return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
diff --git a/buildfarm/filecache.py b/buildfarm/filecache.py
new file mode 100644
index 0000000..52e0183
--- /dev/null
+++ b/buildfarm/filecache.py
@@ -0,0 +1,200 @@
+#!/usr/bin/python
+# Simple database query script for the buildfarm
+#
+# Copyright (C) Jelmer Vernooij <jelmer at samba.org> 2010
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+from buildfarm import (
+ BuildFarm,
+ util,
+ )
+from buildfarm.data import (
+ Build,
+ BuildResultStore,
+ NoSuchBuildError,
+ UploadBuildResultStore,
+ check_dir_exists,
+ )
+
+import os
+
+
+class CachingBuild(Build):
+ """Build subclass that caches some of the results that are expensive
+ to calculate."""
+
+ def __init__(self, store, *args, **kwargs):
+ self._store = store
+ super(CachingBuild, self).__init__(*args, **kwargs)
+ if self.revision:
+ self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler, self.revision)
+ else:
+ self.cache_basename = self._store.cache_fname(self.tree, self.host, self.compiler)
+
+ def revision_details(self):
+ st1 = os.stat("%s.log" % self.basename)
+
+ try:
+ st2 = os.stat("%s.revision" % self.cache_basename)
+ except OSError:
+ # File does not exist
+ st2 = None
+
+ # the ctime/mtime asymmetry is needed so we don't get fooled by
+ # the mtime update from rsync
+ if st2 and st1.st_ctime <= st2.st_mtime:
+ (revid, timestamp) = util.FileLoad("%s.revision" % self.cache_basename).split(":", 2)
+ if timestamp == "":
+ timestamp = None
+ if revid == "":
+ revid = None
+ return (revid, timestamp)
+ (revid, timestamp) = super(CachingBuild, self).revision_details()
+ if not self._store.readonly:
+ util.FileSave("%s.revision" % self.cache_basename, "%s:%s" % (revid, timestamp or ""))
+ return (revid, timestamp)
+
+ def err_count(self):
+ st1 = os.stat("%s.err" % self.basename)
+
+ try:
+ st2 = os.stat("%s.errcount" % self.cache_basename)
+ except OSError:
+ # File does not exist
+ st2 = None
+
+ if st2 and st1.st_ctime <= st2.st_mtime:
+ return util.FileLoad("%s.errcount" % self.cache_basename)
+
+ ret = super(CachingBuild, self).err_count()
+
+ if not self._store.readonly:
+ util.FileSave("%s.errcount" % self.cache_basename, str(ret))
+
+ return ret
+
+ def status(self):
+ cachefile = self.cache_basename + ".status"
+
+ st1 = os.stat("%s.log" % self.basename)
+
+ try:
+ st2 = os.stat(cachefile)
+ except OSError:
+ # No such file
+ st2 = None
+
+ if st2 and st1.st_ctime <= st2.st_mtime:
+ return eval(util.FileLoad(cachefile))
+
+ ret = super(CachingBuild, self).status()
+
+ if not self._store.readonly:
+ util.FileSave(cachefile, repr(ret))
+
+ return ret
+
+
+class CachingUploadBuildResultStore(UploadBuildResultStore):
+
+ def __init__(self, basedir, cachedir, readonly=False):
+ """Open the database.
+
+ :param readonly: Whether to avoid saving cache files
+ """
+ super(CachingUploadBuildResultStore, self).__init__(basedir)
+ self.cachedir = cachedir
+ self.readonly = readonly
+
+ def cache_fname(self, tree, host, compiler):
+ return os.path.join(self.cachedir, "build.%s.%s.%s" % (tree, host, compiler))
+
+ def get_build(self, tree, host, compiler):
+ basename = self.build_fname(tree, host, compiler)
+ logf = "%s.log" % basename
+ if not os.path.exists(logf):
+ raise NoSuchBuildError(tree, host, compiler)
+ return CachingBuild(self, basename, tree, host, compiler)
+
+
+class CachingBuildResultStore(BuildResultStore):
+
+ def __init__(self, basedir, cachedir, readonly=False):
+ super(CachingBuildResultStore, self).__init__(basedir)
+
+ self.cachedir = cachedir
+ check_dir_exists("cache", self.cachedir)
+
+ self.readonly = readonly
+
+ def get_build(self, tree, host, compiler, rev):
+ basename = self.build_fname(tree, host, compiler, rev)
+ logf = "%s.log" % basename
+ if not os.path.exists(logf):
+ raise NoSuchBuildError(tree, host, compiler, rev)
+ return CachingBuild(self, basename, tree, host, compiler, rev)
+
+ def cache_fname(self, tree, host, compiler, rev):
+ return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
+
+
+class CachingBuildFarm(BuildFarm):
+
+ def __init__(self, path=None, readonly=False, cachedirname=None):
+ self._cachedirname = cachedirname
+ self.readonly = readonly
+ super(CachingBuildFarm, self).__init__(path)
+
+ def _get_cachedir(self):
+ if self._cachedirname is not None:
+ return os.path.join(self.path, self._cachedirname)
+ else:
+ return os.path.join(self.path, "cache")
+
+ def _open_build_results(self):
+ return CachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
+ self._get_cachedir(), readonly=self.readonly)
+
+ def _open_upload_build_results(self):
+ return CachingUploadBuildResultStore(os.path.join(self.path, "data", "upload"),
+ self._get_cachedir(), readonly=self.readonly)
+
+ def lcov_status(self, tree):
+ """get status of build"""
+ cachefile = os.path.join(self._get_cachedir(),
+ "lcov.%s.%s.status" % (self.LCOVHOST, tree))
+ file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
+ try:
+ st1 = os.stat(file)
+ except OSError:
+ # File does not exist
+ raise NoSuchBuildError(tree, self.LCOVHOST, "lcov")
+ try:
+ st2 = os.stat(cachefile)
+ except OSError:
+ # file does not exist
+ st2 = None
+
+ if st2 and st1.st_ctime <= st2.st_mtime:
+ ret = util.FileLoad(cachefile)
+ if ret == "":
+ return None
+ return ret
+
+ perc = super(CachingBuildFarm, self).lcov_status(tree)
+ if not self.readonly:
+ util.FileSave(cachefile, perc)
+ return perc
diff --git a/buildfarm/history.py b/buildfarm/history.py
index 383782b..94bf756 100644
--- a/buildfarm/history.py
+++ b/buildfarm/history.py
@@ -33,13 +33,11 @@ class Branch(object):
"""Determine all authors that have contributed to this project.
"""
ret = set()
- for i, rev in enumerate(self.log()):
- if i == limit:
- break
+ for rev in self.log(limit):
ret.add(rev.author)
return ret
- def log(self):
+ def log(self, limit=None):
raise NotImplementedError(self.log)
def diff(self, revision):
@@ -48,11 +46,12 @@ class Branch(object):
class Revision(object):
- def __init__(self, revision, date, author, message, modified=[], added=[],
+ def __init__(self, revision, date, committer, author, message, modified=[], added=[],
removed=[]):
self.revision = revision
self.date = date
self.author = author
+ self.committer = committer
self.message = message
self.modified = modified
self.added = added
@@ -84,11 +83,12 @@ class GitBranch(Branch):
removed.add(oldpath)
else:
modified.add(newpath)
- return Revision(commit.id, commit.commit_time, commit.author,
- commit.message, modified=modified, removed=removed,
+ return Revision(commit.id, commit.commit_time,
+ committer=commit.committer, author=commit.author,
+ message=commit.message, modified=modified, removed=removed,
added=added)
- def log(self, from_rev=None, exclude_revs=None):
+ def log(self, from_rev=None, exclude_revs=None, limit=None):
if from_rev is None:
try:
commit = self.repo["refs/heads/%s" % self.branch]
@@ -104,6 +104,8 @@ class GitBranch(Branch):
commit = self.repo[commit_id]
yield self._revision_from_commit(commit)
done.add(commit.id)
+ if len(done) >= limit:
+ return
# FIXME: Add sorted by commit_time
for p in commit.parents:
if exclude_revs is not None and p in exclude_revs:
diff --git a/buildfarm/tests/test_buildfarm.py b/buildfarm/tests/test_buildfarm.py
index 413c44c..9a62a7c 100644
--- a/buildfarm/tests/test_buildfarm.py
+++ b/buildfarm/tests/test_buildfarm.py
@@ -17,7 +17,6 @@
from buildfarm import (
BuildFarm,
- CachingBuildFarm,
data,
read_trees_from_conf,
)
@@ -121,10 +120,4 @@ class BuildFarmTests(BuildFarmTestBase, BuildFarmTestCase):
self.x = BuildFarm(self.path)
-class CachingBuildFarmTests(BuildFarmTestBase, BuildFarmTestCase):
-
- def setUp(self):
- BuildFarmTestCase.setUp(self)
- BuildFarmTestBase.setUp(self)
- self.x = CachingBuildFarm(self.path)
diff --git a/buildfarm/tests/test_data.py b/buildfarm/tests/test_data.py
index fe94402..5c233c2 100755
--
build.samba.org
More information about the samba-cvs
mailing list