[PATCH 13/14] pytdb: Python tdbtorture port
Kirill Smelkov
kirr at landau.phys.spbu.ru
Sat Oct 2 07:43:52 MDT 2010
This adds more runtime testing to pytdb bindings, to ensure we don't
have segfaults etc, and also now we'll have more testing coverage
than python/test/simple.py.
NOTE: because in python errors raise exceptions, we can avoid
lot's of manual checking after many tdb calls, and simply rely on that
exceptions being propagated to upper level and reported there.
Signed-off-by: Kirill Smelkov <kirr at landau.phys.spbu.ru>
---
lib/tdb/python/pytdbtorture | 312 +++++++++++++++++++++++++++++++++++++++++++
lib/tdb/tdb.mk | 3 +-
2 files changed, 314 insertions(+), 1 deletions(-)
create mode 100755 lib/tdb/python/pytdbtorture
diff --git a/lib/tdb/python/pytdbtorture b/lib/tdb/python/pytdbtorture
new file mode 100755
index 0000000..f9056cb
--- /dev/null
+++ b/lib/tdb/python/pytdbtorture
@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+"""Python port of tdbtorture.c"""
+
+from os import read, write, pipe, close, unlink, O_RDWR, O_CREAT
+from os import kill, getpid, fork, waitpid, system, \
+ WNOHANG, WIFSIGNALED, WTERMSIG, WEXITSTATUS
+from signal import signal, SIGTERM, SIGUSR1, SIGUSR2, SIG_IGN
+from time import time, sleep
+from random import random as drand, seed as srandom
+from getopt import getopt, GetoptError
+import sys
+from sys import exit, stdout, stderr
+import os.path
+import struct
+
+import tdb
+
+
+# probabilities of things to do
+REOPEN_PROB = 30
+DELETE_PROB = 8
+STORE_PROB = 4
+APPEND_PROB = 6
+TRANSACTION_PROB = 10
+TRANSACTION_PREPARE_PROB = 2
+LOCKSTORE_PROB = 5
+TRAVERSE_PROB = 20
+TRAVERSE_READ_PROB = 20
+CULL_PROB = 100
+
+KEYLEN = 3
+DATALEN = 100
+
+db = None
+in_transaction = 0
+error_count = 0
+always_transaction = 0
+hash_size = 2
+loopnum = 0
+count_pipe = None
+
+
+def die(msg):
+ print >>stderr, msg
+ exit(1)
+
+def rand():
+ # 10x faster than randint(0,0xffffffff)
+ return int(drand() * 0xffffffff)
+
+def everyn(n):
+ # True every n'th call, (n=0 -- never)
+ return (n != 0) and ((rand() % n) == 0)
+
+
+def tdb_log(db, level, msg):
+ global error_count
+
+ if level != tdb.DEBUG_TRACE:
+ error_count += 1
+
+ print msg,
+ stdout.flush()
+
+ if 0 and level != tdb.DEBUG_TRACE:
+ signal(SIGUSR1, SIG_IGN)
+ system("xterm -e gdb /proc/%d/exe %d" % (getpid(), getpid()))
+
+
+
+def randbuf(n):
+ buf = ''.join( [chr(ord('a') + (rand() % 26)) for _ in range(n)] )
+ return buf
+
+
+def cull_traverse(db, key, data):
+ if everyn(CULL_PROB):
+ db.delete(key)
+
+ return True
+
+
+def addrec_db():
+ global in_transaction
+
+ key = randbuf(1 + (rand() % KEYLEN))
+ data = randbuf(1 + (rand() % DATALEN))
+
+ if in_transaction == 0 and everyn(REOPEN_PROB):
+ db.reopen() # TODO reopen -> reopen_all
+
+ elif in_transaction == 0 and (always_transaction or everyn(TRANSACTION_PROB)):
+ db.transaction_start()
+ in_transaction += 1
+
+ elif in_transaction and everyn(TRANSACTION_PROB):
+ if everyn(TRANSACTION_PREPARE_PROB):
+ db.transaction_prepare_commit()
+ db.transaction_commit()
+ in_transaction -= 1
+
+ elif in_transaction and everyn(TRANSACTION_PROB):
+ db.transaction_cancel()
+ in_transaction -= 1
+
+ elif everyn(DELETE_PROB):
+ try:
+ db.delete(key)
+ except RuntimeError:
+ pass # it's ok if key is not exists yet
+ # XXX -> KeyError ?
+
+ elif everyn(STORE_PROB):
+ db.store(key, data, tdb.REPLACE)
+
+ elif everyn(APPEND_PROB):
+ db.append(key, data)
+
+ elif everyn(LOCKSTORE_PROB):
+ db.chainlock(key)
+ try:
+ data = db[key]
+ except KeyError:
+ pass # it's ok if there is no such key yet
+ db.store(key, data, tdb.REPLACE)
+ db.chainunlock(key)
+
+ elif everyn(TRAVERSE_PROB):
+ db.traverse(cull_traverse)
+
+ elif everyn(TRAVERSE_READ_PROB):
+ db.traverse_read(None)
+
+ else:
+ try:
+ data = db[key] # tdb_fetch
+ except KeyError:
+ pass # it's ok if there is no such key yet
+
+
+def traverse_fn(db, key, data):
+ db.delete(key)
+ return True
+
+def usage():
+ print "Usage: pytdbtorture [-t] [-k] [-n NUM_PROCS] [-l NUM_LOOPS] [-s SEED] [-H HASH_SIZE]"
+ exit(0)
+
+def send_count_and_suicide(sig, frame):
+ # This ensures our successor can continue where we left off.
+ xloopnum = struct.pack('I', loopnum)
+ write(count_pipe, xloopnum)
+ # This gives a unique signature.
+ kill(getpid(), SIGUSR2)
+
+
+def run_child(i, seed, num_loops, start):
+ global db, in_transaction, loopnum
+
+ db = tdb.Tdb("pytorture.tdb", hash_size, tdb.DEFAULT,
+ O_RDWR | O_CREAT, 0600, tdb_log)
+
+ srandom(seed)
+
+ # Set global, then we're ready to handle being killed.
+ loopnum = start;
+ signal(SIGUSR1, send_count_and_suicide);
+
+ while loopnum < num_loops and error_count == 0:
+ addrec_db()
+ loopnum += 1
+
+ if error_count == 0:
+ db.traverse_read(None)
+ if always_transaction:
+ while in_transaction:
+ db.transaction_cancel()
+ in_transaction -= 1
+ db.transaction_start()
+
+ db.traverse(traverse_fn)
+ db.traverse(traverse_fn)
+ if (always_transaction):
+ db.transaction_commit()
+
+ db.close()
+
+ return min(error_count, 100)
+
+
+def tortureit(num_procs, num_loops, seed, kill_random):
+ global error_count, count_pipe
+
+ pids = [0] * num_procs
+ done = [0] * num_procs
+
+ pfds = pipe()
+ count_pipe = pfds[1]
+
+ for i in range(num_procs):
+ pids[i] = fork()
+ if pids[i] == 0:
+ close(pfds[0])
+ if i == 0:
+ print "Testing with %d processes, %d loops, %d hash_size, seed=%d%s" % \
+ (num_procs, num_loops, hash_size, seed,
+ always_transaction and " (all with transactions)" or "")
+
+ exit( run_child(i, seed, num_loops, 0) )
+
+
+ while num_procs:
+ if error_count != 0:
+ # try and stop the test on any failure
+ for j in range(num_procs):
+ if pids[j] != 0:
+ kill(pids[j], SIGTERM)
+
+ pid, status = waitpid(-1, kill_random and WNOHANG or 0)
+ if pid == 0:
+ # Sleep for 1/10 second.
+ sleep(0.1)
+
+ # Kill someone.
+ kill(pids[rand() % num_procs], SIGUSR1)
+ continue
+
+ try:
+ j = pids.index(pid)
+ except ValueError:
+ die("unknown child %d exited!?" % pid)
+
+
+ if WIFSIGNALED(status):
+ if WTERMSIG(status) in (SIGUSR2, SIGUSR1):
+ # SIGUSR2 means they wrote to pipe.
+ if WTERMSIG(status) == SIGUSR2:
+ xdone = read(pfds[0], struct.calcsize('I'))
+ done[j] = struct.unpack('I', xdone)[0]
+
+ pids[j] = fork()
+ if pids[j] == 0:
+ exit( run_child(j, seed, num_loops, done[j]) )
+
+ print "Restarting child %i for %u-%u" % (j, done[j], num_loops)
+ continue
+
+ print "child %d exited with signal %d" % (pid, WTERMSIG(status))
+ error_count += 1
+
+ elif WEXITSTATUS(status) != 0:
+ print "child %d exited with status %d" % (pid, WEXITSTATUS(status))
+ error_count += 1
+
+ del pids[j]
+ num_procs -= 1
+
+
+def main():
+ global db, error_count, hash_size, always_transaction
+
+ seed = -1
+ num_loops = 5000
+ num_procs = 3
+ kill_random = 0
+
+ try:
+ opts, args = getopt(sys.argv[1:], "n:l:s:H:thk")
+ except GetoptError, e:
+ usage()
+
+ for o, a in opts:
+ if o == '-n':
+ num_procs = int(a)
+ elif o == '-l':
+ num_loops = int(a)
+ elif o == '-H':
+ hash_size = int(a)
+ elif o == '-s':
+ seed = int(a)
+ elif o == '-t':
+ always_transaction = 1
+ elif o == '-k':
+ kill_random = 1
+ else:
+ usage()
+
+ if os.path.lexists("pytorture.tdb"):
+ unlink("pytorture.tdb")
+
+ if seed == -1:
+ seed = int(getpid() + time()) & 0x7FFFFFFF
+
+ if num_procs == 1 and not kill_random:
+ # Don't fork for this case, makes debugging easier.
+ error_count = run_child(0, seed, num_loops, 0)
+ else:
+ tortureit(num_procs, num_loops, seed, kill_random)
+
+
+ if error_count == 0:
+ db = tdb.Tdb("pytorture.tdb", hash_size, tdb.DEFAULT, O_RDWR, 0, tdb_log)
+ db.check(None)
+ db.close()
+ print "OK"
+
+
+ return error_count
+
+
+if __name__ == '__main__':
+ exit( main() )
diff --git a/lib/tdb/tdb.mk b/lib/tdb/tdb.mk
index 64d7996..85a38fc 100644
--- a/lib/tdb/tdb.mk
+++ b/lib/tdb/tdb.mk
@@ -39,7 +39,7 @@ abi_checks::
@./script/abi_checks.sh tdb include/tdb.h
clean::
- rm -f test.db test.tdb torture.tdb test.gdbm
+ rm -f test.db test.tdb torture.tdb pytorture.tdb test.gdbm
rm -f $(TDB_SONAME) $(TDB_SOLIB) $(TDB_STLIB) libtdb.$(SHLIBEXT)
rm -f $(ALL_PROGS) tdb.pc
rm -f tdb.exports.sort tdb.exports.check tdb.exports.check.sort
@@ -75,6 +75,7 @@ install-python:: build-python
check-python:: build-python $(TDB_SONAME)
$(LIB_PATH_VAR)=. PYTHONPATH=".:$(tdbdir)" $(PYTHON) $(tdbdir)/python/tests/simple.py
+ $(LIB_PATH_VAR)=. PYTHONPATH=".:$(tdbdir)" $(PYTHON) $(tdbdir)/python/pytdbtorture
clean::
rm -f tdb.$(SHLIBEXT)
--
1.7.3.1.50.g1e633
More information about the samba-technical
mailing list