[SCM] build.samba.org - branch master updated

Jelmer Vernooij jelmer at samba.org
Mon Nov 1 03:31:07 MDT 2010


The branch, master has been updated
       via  a7d0624 Add python equivalent of build.pl.
       via  c432e2f Make internal functionality private.
       via  9b9cabc Add python version of history module.
      from  dfe4f79 renamed the file

http://gitweb.samba.org/?p=build-farm.git;a=shortlog;h=master


- Log -----------------------------------------------------------------
commit a7d0624a8d666d9476c39ce74d2f0a244446684d
Author: Jelmer Vernooij <jelmer at samba.org>
Date:   Mon Nov 1 10:31:12 2010 +0100

    Add python equivalent of build.pl.

commit c432e2f2e0cd422e0def6a4dd4be5bb2ce1ed5f4
Author: Jelmer Vernooij <jelmer at samba.org>
Date:   Mon Nov 1 10:30:29 2010 +0100

    Make internal functionality private.

commit 9b9cabc8c9235d12e054f0e1d373472085446d1b
Author: Jelmer Vernooij <jelmer at samba.org>
Date:   Mon Nov 1 02:04:25 2010 +0100

    Add python version of history module.

-----------------------------------------------------------------------

Summary of changes:
 buildfarm/history.py            |  188 ++++++++
 buildfarm/tests/test_history.py |   20 +
 web/build.py                    |  925 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 1133 insertions(+), 0 deletions(-)
 create mode 100644 buildfarm/history.py
 create mode 100644 buildfarm/tests/test_history.py
 create mode 100755 web/build.py


Changeset truncated at 500 lines:

diff --git a/buildfarm/history.py b/buildfarm/history.py
new file mode 100644
index 0000000..520c716
--- /dev/null
+++ b/buildfarm/history.py
@@ -0,0 +1,188 @@
+#!/usr/bin/python
+# script to show recent checkins in cvs / svn / git
+#
+# Copyright (C) Andrew Tridgell <tridge at samba.org>     2001
+# Copyright (C) Martin Pool <mbp at samba.org>            2003
+# Copyright (C) Jelmer Vernooij <jelmer at samba.org>     2007-2010
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+
+from buildfarm import util
+
+import commands
+import os
+import time
+
+BASEDIR = "/home/build/master"
+HISTORYDIR = "/home/build/master/cache"
+TIMEZONE = "PST"
+TIMEOFFSET = 0
+UNPACKED_DIR = "/home/ftp/pub/unpacked"
+
+CVSWEB_BASE = "http://pserver.samba.org/cgi-bin/cvsweb"
+VIEWCVS_BASE = "http://websvn.samba.org/cgi-bin/viewcvs.cgi"
+UNPACKED_BASE = "http://svn.samba.org/ftp/unpacked"
+GITWEB_BASE = "http://gitweb.samba.org"
+
+
+class History(object):
+
+    def __init__(self, db):
+        self.db = db
+
+    def _log(self, tree):
+        return util.LoadStructure(os.path.join(HISTORYDIR, "history.%s" % tree))
+
+    def diff(self, author, date, tree, revision):
+        """get recent cvs/svn entries"""
+        # validate the tree
+        t = self.db.trees[tree]
+
+        if t["scm"] == "cvs":
+            self._cvs_diff(t, author, date, tree)
+        elif t["scm"] == "svn":
+            self._svn_diff(t, revision, tree)
+        elif t["scm"] == "git":
+            self._git_diff(t, revision, tree)
+        else:
+            raise Exception("Unknown VCS %s" % t["scm"])
+
+    def _svn_diff(self, t, revision, tree):
+        """show recent svn entries"""
+
+        os.chdir(os.path.join(UNPACKED_DIR, tree))
+
+        # determine the most recent version known to this database
+        for l in commands.getoutput("svn info").splitlines():
+            if l.startswith("Revision"):
+                current_revision = l.strip().split(":")
+                break
+        else:
+            raise Exception("Unable to find current revision")
+
+        if (not revision.isdigit() or int(revision) < 0 or
+            int(revision) > int(current_revision)):
+            raise Exception("unknown revision[%s]" % revision)
+
+        log = self._log(tree)
+
+        # backwards? why? well, usually our users are looking for the newest
+        # stuff, so it's most likely to be found sooner
+        for i in range(len(log), 0, -1):
+            if log[i]["REVISION"] == revision:
+                entry = log[i]
+                break
+        else:
+            raise Exception("Unable to locate commit information revision[%s]." % revision)
+
+        # get information about the current diff
+        title = "SVN Diff in %s:%s for revision r%s" % (
+            tree, t["branch"], revision)
+
+        old_revision = revision - 1
+        cmd = "svn diff -r %s:%s" % (old_revision, revision)
+
+        return (title, entry, tree, [(cmd, commands.getoutput("%s 2> /dev/null" % cmd))])
+
+    def _cvs_diff(self, t, author, date, tree):
+        """show recent cvs entries"""
+        os.chdir(os.path.join(UNPACKED_DIR, tree))
+
+        log = self._log(tree)
+
+        # for paranoia, check that the date string is a valid date
+        if not date[0].isdigit():
+            raise Exception("unknown date")
+
+        for i in range(log):
+            if author == log[i]["AUTHOR"] and date == log[i]["DATE"]:
+                entry = log[i]
+                break
+        else:
+            raise Exception("Unable to locate commit information author[%s] data[%s]." % (
+                author, date))
+
+        t1 = time.ctime(date-60+(TIMEOFFSET*60*60)).strip()
+        t2 = time.ctime(date+60+(TIMEOFFSET*60*60)).strip()
+
+        title = "CVS Diff in %s:%s for %s" % (tree, t["branch"], t1)
+
+        if entry["TAG"] != "" and entry["REVISIONS"] != "":
+            raise Exception("sorry, cvs diff on branches not currently possible due to a limitation in cvs")
+
+        os.environ['CVS_PASSFILE'] = os.path.join(BASEDIR, ".cvspass")
+
+        if entry["REVISIONS"]:
+            diffs = []
+            for f in entry["REVISIONS"].keys():
+                if entry["REVISIONS"][f]["REV1"] == "NONE":
+                    cmd = "cvs rdiff -u -r 0 -r %s %s" % (entry["REVISIONS"][f]["REV2"], f)
+                elif entry["REVISIONS"][f]["REV2"] == "NONE":
+                    cmd = "cvs rdiff -u -r %s -r 0 %s" % (
+                        entry["REVISIONS"][f]["REV1"], f)
+                else:
+                    cmd = "cvs diff -b -u -r %s -r %s %s" % (
+                        entry["REVISIONS"][f]["REV1"], entry["REVISIONS"][f]["REV2"], f)
+
+                diffs.append((cmd, commands.getoutput("%s 2> /dev/null" % cmd)))
+        else:
+            cmd = "cvs diff -b -u -D \"%s %s\" -D \"%s %s\" %s" % (
+                t1, TIMEZONE, t2, TIMEZONE, entry["FILES"])
+
+            diffs = [(cmd, commands.getoutput("%s 2> /dev/null" % cmd))]
+        return (title, entry, tree, diffs)
+
+    def _git_diff(self, t, revision, tree):
+        """show recent git entries"""
+        os.chdir(os.path.join(UNPACKED_DIR, tree))
+
+        log = self._log(tree)
+
+        # backwards? why? well, usually our users are looking for the newest
+        # stuff, so it's most likely to be found sooner
+        for i in range(len(log), 0, -1):
+            if log[i]["REVISION"] == revision:
+                entry = log[i]
+                break
+        else:
+            raise Exception("Unable to locate commit information revision[%s]." % revision)
+
+        # get information about the current diff
+        title = "GIT Diff in %s:%s for revision %s" % (
+            tree, t["branch"], revision)
+
+        cmd = "git diff %s^ %s ./" % (revision, revision)
+        return (title, entry, tree, [(cmd, commands.getoutput("%s 2> /dev/null" % cmd))])
+
+    def authors(self, tree):
+        log = self._log(tree)
+        authors = set()
+        for entry in log:
+            authors.add(entry["AUTHOR"])
+        return authors
+
+    def history(self, tree, author=None):
+        """get commit history for the given tree"""
+        log = self._log(tree)
+
+        # what? backwards? why is that? oh... I know... we want the newest first
+        for i in range(len(log), 0, -1):
+            entry = log[i]
+            if (author is None or
+                (author == "") or
+                (author == "ALL") or
+                (author == entry["AUTHOR"])):
+                yield entry, tree
diff --git a/buildfarm/tests/test_history.py b/buildfarm/tests/test_history.py
new file mode 100644
index 0000000..8945615
--- /dev/null
+++ b/buildfarm/tests/test_history.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+# Copyright (C) Jelmer Vernooij <jelmer at samba.org> 2010
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 3 of the License, or
+#   (at your option) any later version.
+#   
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#   
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+import unittest
+
+from buildfarm import history
diff --git a/web/build.py b/web/build.py
new file mode 100755
index 0000000..dd720d4
--- /dev/null
+++ b/web/build.py
@@ -0,0 +1,925 @@
+#!/usr/bin/python
+# This CGI script presents the results of the build_farm build
+#
+# Copyright (C) Andrew Tridgell <tridge at samba.org>     2001-2005
+# Copyright (C) Andrew Bartlett <abartlet at samba.org>   2001
+# Copyright (C) Vance Lankhaar  <vance at samba.org>      2002-2005
+# Copyright (C) Martin Pool <mbp at samba.org>            2001
+# Copyright (C) Jelmer Vernooij <jelmer at samba.org>     2007-2010
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 3 of the License, or
+#   (at your option) any later version.
+#   
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#   
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# TODO: Allow filtering of the "Recent builds" list to show
+# e.g. only broken builds or only builds that you care about.
+
+from buildfarm import data, util, history
+
+import cgi
+import os
+import re
+import time
+
+# Show tracebacks on errors
+import cgitb
+cgitb.enable()
+
+webdir = os.path.dirname(__file__)
+basedir = os.path.abspath(os.path.join(webdir, ".."))
+
+db = data.BuildfarmDatabase(basedir)
+form = cgi.FieldStorage()
+history = history.History(db)
+
+compilers = db.compilers
+hosts = db.hosts
+trees = db.trees
+OLDAGE = db.OLDAGE
+
+# this is automatically filled in
+deadhosts = []
+
+def cgi_headers():
+    """start CGI headers"""
+    print "Content-Type: text/html\n"
+
+    print "<html>"
+    print "  <head>"
+    print "    <title>samba.org build farm</title>"
+    print "    <script language='javascript' src='/build_farm.js'/>"
+    print "    <meta name='keywords' contents='Samba SMB CIFS Build Farm'/>"
+    print "    <meta name='description' contents='Home of the Samba Build Farm, the automated testing facility.'/>"
+    print "    <meta name='robots' contents='noindex'/>"
+    print "    <link rel='stylesheet' href='/build_farm.css' type='text/css' media='all'/>"
+    print "    <link rel='stylesheet' href='http://master.samba.org/samba/style/common.css' type='text/css' media='all'/>"
+    print "    <link rel='shortcut icon' href='http://www.samba.org/samba/images/favicon.ico'/>"
+    print "  </head>"
+    print "<body>"
+
+    print util.FileLoad(os.path.join(webdir, "header2.html"))
+    main_menu()
+    print util.FileLoad(os.path.join(webdir, "header3.html"))
+
+
+def cgi_footers():
+    """end CGI"""
+    print util.FileLoad(os.path.join(webdir, "footer.html"))
+    print "</body>"
+    print "</html>"
+
+
+def get_param(param):
+    """get a param from the request, after sanitizing it"""
+    if param not in form:
+        return None
+
+    result = [s.replace(" ", "_") for s in form.getlist(param)]
+
+    for entry in result:
+        if re.match("[^a-zA-Z0-9\-\_\.]", entry):
+            raise Exception("Parameter %s is invalid" % param)
+
+    return result[0]
+
+
+def build_link(host, tree, compiler, rev, status):
+    if rev:
+        opt_rev = ';revision=%s' % rev
+    else:
+        opt_rev = ''
+    return "<a href='%s?function=View+Build;host=%s;tree=%s;compiler=%s%s'>%s</a>" % (myself, host, tree, compiler, opt_rev, status)
+
+
+def build_status(host, tree, compiler, rev):
+    status = db.build_status(host, tree, compiler, rev)
+    return build_link(host, tree, compiler, rev, status)
+
+
+def host_age(host):
+    """get the overall age of a host"""
+    ret = -1
+    for compiler in compilers:
+        for tree in trees:
+            age = db.build_age_mtime(host, tree, compiler, "")
+            if age != -1 and (age < ret or ret == -1):
+                ret = age
+    return ret
+
+
+def red_age(age):
+    """show an age as a string"""
+    if age > OLDAGE:
+        return "<span clsas='old'>%s</span>" % util.dhm_time(age)
+    return util.dhm_time(age)
+
+
+def build_status_vals(status):
+    """translate a status into a set of int representing status"""
+    status = util.strip_html(status)
+
+    status = status.replace("ok", "0")
+    status = status.replace("?", "0")
+    status = status.replace("PANIC", "1")
+
+    return status.split("/")
+
+
+def view_summary(output_type):
+    """view build summary"""
+    i = 0
+    cols = 2
+    broken = 0
+    broken_count = {}
+    panic_count = {}
+    host_count = {}
+
+    # zero broken and panic counters
+    for tree in trees:
+        broken_count[tree] = 0
+        panic_count[tree] = 0
+        host_count[tree] = 0
+
+    # set up a variable to store the broken builds table's code, so we can
+    # output when we want
+    broken_table = ""
+    last_host = ""
+
+    # for the text report, include the current time
+    if output_type == 'text':
+        t = time.gmtime()
+        print "Build status as of %s\n\n" % t
+
+    for host in hosts:
+        for compiler in compilers:
+            for tree in trees:
+                status = build_status(host, tree, compiler, "")
+                if status.startswith("Unknown Build"):
+                    continue
+                age_mtime = db.build_age_mtime(host, tree, compiler, "")
+
+                if age_mtime != -1:
+                    host_count[tree]+=1
+
+                if "status failed" in status:
+                    broken_count[tree]+=1
+                    if "PANIC" in status:
+                        panic_count[tree]+=1
+
+    if output_type == 'text':
+        print "Build counts:\n"
+        print "%-12s %-6s %-6s %-6s\n" % ("Tree", "Total", "Broken", "Panic")
+    else:
+        print "<div id='build-counts' class='build-section'>"
+        print "<h2>Build counts:</h2>"
+        print "<table class='real'>"
+        print "<thead><tr><th>Tree</th><th>Total</th><th>Broken</th><th>Panic</th><th>Test coverage</th></tr></thead>"
+        print "<tbody>"
+
+    for tree in sorted(trees.keys()):
+        if output_type == 'text':
+            print "%-12s %-6s %-6s %-6s\n" % (tree, host_count[tree],
+                    broken_count[tree], panic_count[tree])
+        else:
+            print "<tr>"
+            print "<td>%s</td>" % tree_link(tree)
+            print "<td>%s</td>" % host_count[tree]
+            print "<td>%s</td>" % broken_count[tree]
+            if panic_count[tree]:
+                    print "<td class='panic'>"
+            else:
+                    print "<td>"
+            print panic_count[tree] + "</td>"
+            print "<td>%s</td>" % db.lcov_status(tree)
+            print "</tr>"
+
+    if output_type == 'text':
+        print "\n"
+    else:
+        print "</tbody></table>"
+        print "</div>"
+
+
+def revision_link(revision, tree):
+    """return a link to a particular revision"""
+
+    revision = revision.lstrip()
+    if revision == "0":
+        return "0"
+
+    rev_short = revision
+    if len(revision) == 40:
+        rev_short = re.sub("(^.{7}).*", "\\1(git)", rev_short)
+
+    return "<a href='%s?function=diff;tree=%s;revision=%s' title='View Diff for %s'>%s</a>" % (myself, tree, revision, revision, rev_short)
+
+
+def tree_link(tree):
+    # return a link to a particular tree
+    branch = ""
+    if tree in trees:
+        branch = ":%s" % trees[tree]["branch"]
+
+    return "<a href='%s?function=Recent+Builds;tree=%s' title='View recent builds for %s'>%s%s</a>" % (myself, tree, tree, tree, branch)
+
+
+def view_recent_builds(tree, sort_by):
+    """Draw the "recent builds" view"""
+    i = 0
+    cols = 2
+    broken = 0
+    last_host = ""
+    all_builds = []
+
+    def status_cmp(a, b):
+        bstat = build_status_vals(b)
+        astat = build_status_vals(a)
+
+        # handle panic
+        if len(bstat) > 4 and bstat[4]:
+            return 1
+        elif len(astat) > 4 and astat[4]:
+            return -1
+
+        return (cmp(astat[0], bstat[0]) or # configure
+                cmp(astat[1], bstat[1]) or # compile
+                cmp(astat[2], bstat[2]) or # install
+                cmp(astat[3], bstat[3])) # test
+
+    cmp_funcs = {
+        "revision": lambda a, b: cmp(a[7], b[7]),
+        "age": lambda a, b: cmp(a[0], b[0]),
+        "host": lambda a, b: cmp(a[2], b[2]),
+        "platform": lambda a, b: cmp(a[1], b[1]),
+        "compiler": lambda a, b: cmp(a[3], b[3]),
+        "status": lambda a, b: status_cmp(a[5], b[5]),
+        }
+
+    assert tree in trees, "not a build tree"
+    assert sort_by in sort, "not a valid sort"
+
+    t = trees[tree]
+
+    for host in hosts:
+        for compiler in compilers:


-- 
build.samba.org


More information about the samba-cvs mailing list