>From 951fdb8e9dceaa1b2a1813a5dad1d694281e6bd9 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Wed, 17 Aug 2016 10:56:39 +1200 Subject: [PATCH 1/3] Add AD DC performance tests These test a variety of simple AD DC operations. These tests are NOT independent of each other and must be run in the right order (alphabetically, which is guaranteed by Python's unittest module) -- the running of each test is part of the set-up for later modules. This means we have to subvert unittest a bit, but it saves hours of repeated set-up. These tests are not intended to push edge cases, but to hammer common operations that should work on all versions of Samba. The tests have been tested back to Samba 4.0.26. Signed-off-by: Douglas Bagnall --- source4/dsdb/tests/python/ad_dc_performance.py | 338 +++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 source4/dsdb/tests/python/ad_dc_performance.py diff --git a/source4/dsdb/tests/python/ad_dc_performance.py b/source4/dsdb/tests/python/ad_dc_performance.py new file mode 100644 index 0000000..e811ef3 --- /dev/null +++ b/source4/dsdb/tests/python/ad_dc_performance.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import optparse +import sys +sys.path.insert(0, 'bin/python') + +import os +import samba +import samba.getopt as options +import random +import tempfile +import shutil +import time + +from samba.netcmd.main import cmd_sambatool + +# We try to use the test infrastructure of Samba 4.3+, but if it +# doesn't work, we are probably in a back-ported patch and trying to +# run on 4.1 or something. +# +# Don't copy this horror into ordinary tests -- it is special for +# performance tests that want to apply to old versions. +try: + from samba.tests.subunitrun import SubunitOptions, TestProgram + ANCIENT_SAMBA = False +except ImportError: + ANCIENT_SAMBA = True + samba.ensure_external_module("testtools", "testtools") + samba.ensure_external_module("subunit", "subunit/python") + from subunit.run import SubunitTestRunner + import unittest + +from samba.samdb import SamDB +from samba.auth import system_session +from ldb import Message, MessageElement, Dn, LdbError +from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE +from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL + +parser = optparse.OptionParser("ad_dc_performance.py [options] ") +sambaopts = options.SambaOptions(parser) +parser.add_option_group(sambaopts) +parser.add_option_group(options.VersionOptions(parser)) + +if not ANCIENT_SAMBA: + subunitopts = SubunitOptions(parser) + parser.add_option_group(subunitopts) + +# use command line creds if available +credopts = options.CredentialsOptions(parser) +parser.add_option_group(credopts) +opts, args = parser.parse_args() + + +if len(args) < 1: + parser.print_usage() + sys.exit(1) + +host = args[0] + +lp = sambaopts.get_loadparm() +creds = credopts.get_credentials(lp) + +random.seed(1) + + +class PerfTestException(Exception): + pass + + +BATCH_SIZE = 1000 +N_GROUPS = 5 + + +class GlobalState(object): + next_user_id = 0 + n_groups = 0 + next_linked_user = 0 + next_relinked_user = 0 + next_linked_user_3 = 0 + next_removed_link_0 = 0 + + +class UserTests(samba.tests.TestCase): + + def add_if_possible(self, *args, **kwargs): + """In these tests sometimes things are left in the database + deliberately, so we don't worry if we fail to add them a second + time.""" + try: + self.ldb.add(*args, **kwargs) + except LdbError: + pass + + def setUp(self): + super(UserTests, self).setUp() + self.state = GlobalState # the class itself, not an instance + self.lp = lp + self.ldb = SamDB(host, credentials=creds, + session_info=system_session(lp), lp=lp) + self.base_dn = self.ldb.domain_dn() + self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn) + self.ou_users = "OU=users,%s" % self.ou + self.ou_groups = "OU=groups,%s" % self.ou + self.ou_computers = "OU=computers,%s" % self.ou + + for dn in (self.ou, self.ou_users, self.ou_groups, + self.ou_computers): + self.add_if_possible({ + "dn": dn, + "objectclass": "organizationalUnit"}) + + def tearDown(self): + super(UserTests, self).tearDown() + + def test_00_00_do_nothing(self): + # this gives us an idea of the overhead + pass + + def _prepare_n_groups(self, n): + self.state.n_groups = n + for i in range(n): + self.add_if_possible({ + "dn": "cn=g%d,%s" % (i, self.ou_groups), + "objectclass": "group"}) + + def _add_users(self, start, end): + for i in range(start, end): + self.ldb.add({ + "dn": "cn=u%d,%s" % (i, self.ou_users), + "objectclass": "user"}) + + def _test_join(self): + tmpdir = tempfile.mkdtemp() + if '://' in host: + server = host.split('://', 1)[1] + else: + server = host + cmd = cmd_sambatool.subcommands['domain'].subcommands['join'] + result = cmd._run("samba-tool domain join", + creds.get_realm(), + "dc", "-U%s%%%s" % (creds.get_username(), + creds.get_password()), + '--targetdir=%s' % tmpdir, + '--server=%s' % server) + + shutil.rmtree(tmpdir) + + def _test_unindexed_search(self): + expressions = [ + ('(&(objectclass=user)(description=' + 'Built-in account for adminstering the computer/domain))'), + '(description=Built-in account for adminstering the computer/domain)', + '(objectCategory=*)', + '(samaccountname=Administrator)' + ] + for expression in expressions: + t = time.time() + for i in range(10): + self.ldb.search(self.ou, + expression=expression, + scope=SCOPE_SUBTREE, + attrs=['cn']) + print >> sys.stderr, '%d %s took %s' % (i, expression, + time.time() - t) + + def _test_indexed_search(self): + expressions = ['(objectclass=group)', + '(samaccountname=Administrator)' + ] + for expression in expressions: + t = time.time() + for i in range(100): + self.ldb.search(self.ou, + expression=expression, + scope=SCOPE_SUBTREE, + attrs=['cn']) + print >> sys.stderr, '%d runs %s took %s' % (i, expression, + time.time() - t) + + def _test_add_many_users(self, n=BATCH_SIZE): + s = self.state.next_user_id + e = s + n + self._add_users(s, e) + self.state.next_user_id = e + + test_00_00_join_empty_dc = _test_join + + test_00_01_adding_users_1000 = _test_add_many_users + test_00_02_adding_users_2000 = _test_add_many_users + test_00_03_adding_users_3000 = _test_add_many_users + + test_00_10_join_unlinked_dc = _test_join + test_00_11_unindexed_search_3k_users = _test_unindexed_search + test_00_12_indexed_search_3k_users = _test_indexed_search + + def _link_user_and_group(self, u, g): + m = Message() + m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups)) + m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users), + FLAG_MOD_ADD, "member") + self.ldb.modify(m) + + def _unlink_user_and_group(self, u, g): + user = "cn=u%d,%s" % (u, self.ou_users) + group = "CN=g%d,%s" % (g, self.ou_groups) + m = Message() + m.dn = Dn(self.ldb, group) + m["member"] = MessageElement(user, FLAG_MOD_DELETE, "member") + self.ldb.modify(m) + + def _test_link_many_users(self, n=BATCH_SIZE): + self._prepare_n_groups(N_GROUPS) + s = self.state.next_linked_user + e = s + n + for i in range(s, e): + g = i % N_GROUPS + self._link_user_and_group(i, g) + self.state.next_linked_user = e + + test_01_01_link_users_1000 = _test_link_many_users + test_01_02_link_users_2000 = _test_link_many_users + test_01_03_link_users_3000 = _test_link_many_users + + def _test_link_many_users_offset_1(self, n=BATCH_SIZE): + s = self.state.next_relinked_user + e = s + n + for i in range(s, e): + g = (i + 1) % N_GROUPS + self._link_user_and_group(i, g) + self.state.next_relinked_user = e + + test_02_01_link_users_again_1000 = _test_link_many_users_offset_1 + test_02_02_link_users_again_2000 = _test_link_many_users_offset_1 + test_02_03_link_users_again_3000 = _test_link_many_users_offset_1 + + test_02_10_join_partially_linked_dc = _test_join + test_02_11_unindexed_search_partially_linked_dc = _test_unindexed_search + test_02_12_indexed_search_partially_linked_dc = _test_indexed_search + + def _test_link_many_users_3_groups(self, n=BATCH_SIZE, groups=3): + s = self.state.next_linked_user_3 + e = s + n + self.state.next_linked_user_3 = e + for i in range(s, e): + g = (i + 2) % groups + if g not in (i % N_GROUPS, (i + 1) % N_GROUPS): + self._link_user_and_group(i, g) + + test_03_01_link_users_again_1000_few_groups = _test_link_many_users_3_groups + test_03_02_link_users_again_2000_few_groups = _test_link_many_users_3_groups + test_03_03_link_users_again_3000_few_groups = _test_link_many_users_3_groups + + def _test_remove_links_0(self, n=BATCH_SIZE): + s = self.state.next_removed_link_0 + e = s + n + self.state.next_removed_link_0 = e + for i in range(s, e): + g = i % N_GROUPS + self._unlink_user_and_group(i, g) + + test_04_01_remove_some_links_1000 = _test_remove_links_0 + test_04_02_remove_some_links_2000 = _test_remove_links_0 + test_04_03_remove_some_links_3000 = _test_remove_links_0 + + # back to using _test_add_many_users + test_05_01_adding_users_after_links_4000 = _test_add_many_users + + # reset the link count, to replace the original links + def test_06_01_relink_users_1000(self): + self.state.next_linked_user = 0 + self._test_link_many_users() + + test_06_02_link_users_2000 = _test_link_many_users + test_06_03_link_users_3000 = _test_link_many_users + test_06_04_link_users_4000 = _test_link_many_users + test_06_05_link_users_again_4000 = _test_link_many_users_offset_1 + test_06_06_link_users_again_4000_few_groups = _test_link_many_users_3_groups + + test_07_01_adding_users_after_links_5000 = _test_add_many_users + + def _test_link_random_users_and_groups(self, n=BATCH_SIZE, groups=100): + self._prepare_n_groups(groups) + for i in range(n): + u = random.randrange(self.state.next_user_id) + g = random.randrange(groups) + try: + self._link_user_and_group(u, g) + except LdbError: + pass + + test_08_01_link_random_users_100_groups = _test_link_random_users_and_groups + test_08_02_link_random_users_100_groups = _test_link_random_users_and_groups + + test_10_01_unindexed_search_full_dc = _test_unindexed_search + test_10_02_indexed_search_full_dc = _test_indexed_search + test_11_02_join_full_dc = _test_join + + def test_20_01_delete_50_groups(self): + for i in range(self.state.n_groups - 50, self.state.n_groups): + self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups)) + self.state.n_groups -= 50 + + def _test_delete_many_users(self, n=BATCH_SIZE): + e = self.state.next_user_id + s = max(0, e - n) + self.state.next_user_id = s + for i in range(s, e): + self.ldb.delete("cn=u%d,%s" % (i, self.ou_users)) + + test_21_01_delete_users_5000_lightly_linked = _test_delete_many_users + test_21_02_delete_users_4000_lightly_linked = _test_delete_many_users + test_21_03_delete_users_3000 = _test_delete_many_users + + def test_22_01_delete_all_groups(self): + for i in range(self.state.n_groups): + self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups)) + self.state.n_groups = 0 + + test_23_01_delete_users_after_groups_2000 = _test_delete_many_users + test_23_00_delete_users_after_groups_1000 = _test_delete_many_users + + test_24_02_join_after_cleanup = _test_join + + +if "://" not in host: + if os.path.isfile(host): + host = "tdb://%s" % host + else: + host = "ldap://%s" % host + + +if ANCIENT_SAMBA: + runner = SubunitTestRunner() + if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful(): + sys.exit(1) + sys.exit(0) +else: + TestProgram(module=__name__, opts=subunitopts) -- 2.7.4 >From 21345f37258c04afd277180f6464df10e0961718 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Wed, 17 Aug 2016 10:56:50 +1200 Subject: [PATCH 2/3] make perftest: for performance testing This runs a selection of subunit tests and reduces the output to only the time it takes to run each test. The tests are listed in selftest/perf_tests.py. Signed-off-by: Douglas Bagnall --- Makefile | 3 +++ selftest/filter-subunit | 27 +++++++++++++++++--- selftest/perf_tests.py | 26 +++++++++++++++++++ selftest/subunithelper.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++ selftest/wscript | 12 ++++++--- 5 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 selftest/perf_tests.py diff --git a/Makefile b/Makefile index 95681ae..5cc9077 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ uninstall: test: $(WAF) test $(TEST_OPTIONS) +perftest: + $(WAF) test --perf-test $(TEST_OPTIONS) + help: @echo NOTE: to run extended waf options use $(WAF_BINARY) or modify your PATH $(WAF) --help diff --git a/selftest/filter-subunit b/selftest/filter-subunit index 857b842..c3aba73 100755 --- a/selftest/filter-subunit +++ b/selftest/filter-subunit @@ -44,6 +44,8 @@ parser.add_option("--fail-on-empty", default=False, action="store_true", help="Fail if there was no subunit output") parser.add_option("--list", default=False, action="store_true", help="Operate in list mode") +parser.add_option("--perf-test-output", default=False, + action="store_true", help="orientate output for performance measurement") opts, args = parser.parse_args() if opts.list: @@ -51,6 +53,18 @@ if opts.list: sys.stdout.write("%s%s%s\n" % (opts.prefix, l.rstrip(), opts.suffix)) sys.exit(0) +if opts.perf_test_output: + bad_options = [] + for bad_opt in ('fail_immediately', 'strip_passed_output', + 'flapping', 'expected_failures'): + if getattr(opts, bad_opt): + bad_options.append(bad_opt) + if bad_options: + print >>sys.stderr, ("--perf-test-output is incompatible with --%s" % + (', --'.join(x.replace('_', '-') + for x in bad_options))) + sys.exit(1) + if opts.expected_failures: expected_failures = subunithelper.read_test_regexes(opts.expected_failures) else: @@ -76,10 +90,15 @@ def handle_sigint(sig, stack): signal.signal(signal.SIGINT, handle_sigint) out = subunithelper.SubunitOps(sys.stdout) -msg_ops = subunithelper.FilterOps(out, opts.prefix, opts.suffix, expected_failures, - opts.strip_passed_output, - fail_immediately=opts.fail_immediately, - flapping=flapping) + +if opts.perf_test_output: + msg_ops = subunithelper.PerfFilterOps(out, opts.prefix, opts.suffix) +else: + msg_ops = subunithelper.FilterOps(out, opts.prefix, opts.suffix, + expected_failures, + opts.strip_passed_output, + fail_immediately=opts.fail_immediately, + flapping=flapping) try: ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin) diff --git a/selftest/perf_tests.py b/selftest/perf_tests.py new file mode 100644 index 0000000..d49bdf4 --- /dev/null +++ b/selftest/perf_tests.py @@ -0,0 +1,26 @@ +#!/usr/bin/python + +# This script generates a list of testsuites that should be run to +# test Samba performance. +# +# These tests are not intended to exercise aspect of Samba, but +# perform common simple functions or to ascertain performance. +# + +# The syntax for a testsuite is "-- TEST --" on a single line, followed +# by the name of the test, the environment it needs and the command to run, all +# three separated by newlines. All other lines in the output are considered +# comments. + +from selftesthelpers import * + +samba4srcdir = source4dir() +samba4bindir = bindir() + +plantestsuite_loadlist("samba4.ldap.ad_dc_performance.python(ad_dc_ntvfs)", + "ad_dc_ntvfs", + [python, os.path.join(samba4srcdir, + "dsdb/tests/python/ad_dc_performance.py"), + '$SERVER', '-U"$USERNAME%$PASSWORD"', + '--workgroup=$DOMAIN', + '$LOADLIST', '$LISTOPT']) diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py index a3bb30b..6fe0755 100644 --- a/selftest/subunithelper.py +++ b/selftest/subunithelper.py @@ -17,6 +17,7 @@ __all__ = ['parse_results'] +import datetime import re import sys from samba import subunit @@ -429,6 +430,70 @@ class FilterOps(unittest.TestResult): self.fail_immediately = fail_immediately +class PerfFilterOps(unittest.TestResult): + + def progress(self, delta, whence): + pass + + def output_msg(self, msg): + pass + + def control_msg(self, msg): + pass + + def start_testsuite(self, name): + self.suite_has_time = False + + def end_testsuite(self, name, result, reason=None): + pass + + def _add_prefix(self, test): + return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix) + + def time(self, time): + self.latest_time = time + #self._ops.output_msg("found time %s\n" % time) + self.suite_has_time = True + + def get_time(self): + if self.suite_has_time: + return self.latest_time + return datetime.datetime.utcnow() + + def startTest(self, test): + self.seen_output = True + test = self._add_prefix(test) + self.starts[test.id()] = self.get_time() + + def addSuccess(self, test): + test = self._add_prefix(test) + tid = test.id() + if tid not in self.starts: + self._ops.addError(test, "%s succeeded without ever starting!" % tid) + delta = self.get_time() - self.starts[tid] + self._ops.output_msg("elapsed-time: %s: %f\n" % (tid, delta.total_seconds())) + + def addFailure(self, test, err=''): + tid = test.id() + delta = self.get_time() - self.starts[tid] + self._ops.output_msg("failure: %s failed after %f seconds (%s)\n" % + (tid, delta.total_seconds(), err)) + + def addError(self, test, err=''): + tid = test.id() + delta = self.get_time() - self.starts[tid] + self._ops.output_msg("error: %s failed after %f seconds (%s)\n" % + (tid, delta.total_seconds(), err)) + + def __init__(self, out, prefix='', suffix=''): + self._ops = out + self.prefix = prefix or '' + self.suffix = suffix or '' + self.starts = {} + self.seen_output = False + self.suite_has_time = False + + class PlainFormatter(TestsuiteEnabledTestResult): def __init__(self, verbose, immediate, statistics, diff --git a/selftest/wscript b/selftest/wscript index 61ca0bd..52861df 100644 --- a/selftest/wscript +++ b/selftest/wscript @@ -79,6 +79,8 @@ def set_options(opt): action="store_true", dest='SOCKET_WRAPPER_KEEP_PCAP', default=False) gr.add_option('--random-order', dest='RANDOM_ORDER', default=False, action="store_true", help="Run testsuites in random order") + gr.add_option('--perf-test', dest='PERF_TEST', default=False, + action="store_true", help="run performance tests only") def configure(conf): conf.env.SELFTEST_PREFIX = Options.options.SELFTEST_PREFIX @@ -193,9 +195,13 @@ def cmd_testonly(opt): if not os.path.isdir(env.SELFTEST_PREFIX): os.makedirs(env.SELFTEST_PREFIX, int('755', 8)) - env.TESTLISTS = ('--testlist="${PYTHON} ${srcdir}/selftest/tests.py|" ' + - '--testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" ' + - '--testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|"') + if Options.options.PERF_TEST: + env.TESTLISTS = '--testlist="${PYTHON} ${srcdir}/selftest/perf_tests.py|" ' + env.FILTER_OPTIONS = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --perf-test-output' + else: + env.TESTLISTS = ('--testlist="${PYTHON} ${srcdir}/selftest/tests.py|" ' + + '--testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" ' + + '--testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|"') if CONFIG_SET(opt, 'AD_DC_BUILD_IS_ENABLED'): env.SELFTEST_TARGET = "samba" -- 2.7.4 >From fe0540fff07ee20bbac07e48230c66f14aedcc73 Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Thu, 4 Aug 2016 15:35:46 +1200 Subject: [PATCH 3/3] selftest/wscript: format perftest as json This makes it easier to use with common web-based graphing systems. Signed-off-by: Douglas Bagnall --- selftest/format-subunit-json | 54 ++++++++++++++++++++++++++++++++++++++++++++ selftest/wscript | 5 +++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 selftest/format-subunit-json diff --git a/selftest/format-subunit-json b/selftest/format-subunit-json new file mode 100644 index 0000000..d44918c --- /dev/null +++ b/selftest/format-subunit-json @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Copyright (C) 2008-2010 Jelmer Vernooij +# Copyright (C) 2016 Douglas Bagnall +# Published under the GNU GPL, v3 or later + +import optparse +import os +import signal +import sys +import json + +sys.path.insert(0, "bin/python") + + +def json_formatter(src_f, dest_f): + """We're not even pretending to be a TestResult subclass; just read + from stdin and look for elapsed-time tags.""" + results = {} + + for line in src_f: + line = line.strip() + print >>sys.stderr, line + if line[:14] == 'elapsed-time: ': + name, time = line[14:].rsplit(':', 1) + results[name] = float(time) + + json.dump(results, dest_f, + sort_keys=True, indent=2, separators=(',', ': ')) + + +def main(): + parser = optparse.OptionParser("format-subunit-json [options]") + parser.add_option("--verbose", action="store_true", + help="ignored, for compatibility") + parser.add_option("--immediate", action="store_true", + help="ignored, for compatibility") + parser.add_option("--prefix", type="string", default=".", + help="Prefix to write summary.json to") + opts, args = parser.parse_args() + + fn = os.path.join(opts.prefix, "summary.json") + f = open(fn, 'w') + json_formatter(sys.stdin, f) + f.close() + print + print "A JSON file summarising these tests performance found in:" + print " ", fn + + +def handle_sigint(sig, stack): + sys.exit(0) + +signal.signal(signal.SIGINT, handle_sigint) +main() diff --git a/selftest/wscript b/selftest/wscript index 52861df..35442f7 100644 --- a/selftest/wscript +++ b/selftest/wscript @@ -113,7 +113,10 @@ def cmd_testonly(opt): env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER') if not env.SUBUNIT_FORMATTER: - env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate' + if Options.options.PERF_TEST: + env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit-json --prefix=${SELFTEST_PREFIX}' + else: + env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate' env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping' if Options.options.FAIL_IMMEDIATELY: -- 2.7.4