[SCM] Samba Shared Repository - branch master updated
Andreas Schneider
asn at samba.org
Fri Jun 19 11:00:08 UTC 2020
The branch, master has been updated
via e478470f201 python: Run cmdline tools for arbitary docs test in parallel
via a2bc150a31d python: Run cmdline tools for default docs test in parallel
from 990a0fc4a04 ldb_ldap: fix off-by-one increment in lldb_add_msg_attr
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit e478470f20184cdf4210a1abf24d4e226c51d637
Author: Andreas Schneider <asn at samba.org>
Date: Wed Jun 17 11:24:13 2020 +0200
python: Run cmdline tools for arbitary docs test in parallel
Running samba.tests.docs on my machine:
before -> (2m6.952s)
after -> (22.298s)
Signed-off-by: Andreas Schneider <asn at samba.org>
Reviewed-by: Alexander Bokovoy <ab at samba.org>
Autobuild-User(master): Andreas Schneider <asn at cryptomilk.org>
Autobuild-Date(master): Fri Jun 19 10:59:30 UTC 2020 on sn-devel-184
commit a2bc150a31d990fd1dee05c2f6eaf7e1d1797c79
Author: Andreas Schneider <asn at samba.org>
Date: Wed Jun 17 10:04:27 2020 +0200
python: Run cmdline tools for default docs test in parallel
Running samba.tests.docs on my machine:
before -> (3m52.582s)
after -> (2m6.952s)
Signed-off-by: Andreas Schneider <asn at samba.org>
Reviewed-by: Alexander Bokovoy <ab at samba.org>
-----------------------------------------------------------------------
Summary of changes:
python/samba/tests/docs.py | 359 +++++++++++++++++++++++++++------------------
1 file changed, 213 insertions(+), 146 deletions(-)
Changeset truncated at 500 lines:
diff --git a/python/samba/tests/docs.py b/python/samba/tests/docs.py
index 3657d3d85cb..5fb04ab4e2e 100644
--- a/python/samba/tests/docs.py
+++ b/python/samba/tests/docs.py
@@ -25,6 +25,9 @@ import samba.tests
import os
import subprocess
import xml.etree.ElementTree as ET
+import multiprocessing
+import concurrent.futures
+import tempfile
config_h = os.path.join("bin/default/include/config.h")
config_hash = dict()
@@ -50,6 +53,94 @@ class TestCase(samba.tests.TestCaseInTempDir):
parameters.sort()
return message + '\n\n %s' % ('\n '.join(parameters))
+def get_max_worker_count():
+ cpu_count = multiprocessing.cpu_count()
+
+ # Always run two processes in parallel
+ if cpu_count <= 2:
+ return 2
+
+ max_workers = int(cpu_count / 2)
+ if max_workers < 1:
+ return 1
+
+ return max_workers
+
+def check_or_set_smbconf_default(cmdline, topdir, param, default_param):
+ p = subprocess.Popen(cmdline,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=topdir).communicate()
+ result = p[0].decode().upper().strip()
+ if result != default_param.upper():
+ if not (result == "" and default_param == '""'):
+ return result, param, default_param
+
+ return None
+
+def set_smbconf_arbitary(cmdline, topdir, param, param_type, value_to_use):
+ p = subprocess.Popen(cmdline,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=topdir).communicate()
+ result = p[0].decode().upper().strip()
+ if result != value_to_use.upper():
+ # currently no way to distinguish command lists
+ if param_type == 'list':
+ if ", ".join(result.split()) == value_to_use.upper():
+ return None
+
+ # currently no way to identify octal
+ if param_type == 'integer':
+ try:
+ if int(value_to_use, 8) == int(p[0].strip(), 8):
+ return None
+ except:
+ pass
+
+ return result, param, value_to_use
+
+ return None
+
+def set_smbconf_arbitary_opposite(cmdline, topdir, tempdir, section, param, opposite_value, value_to_use):
+ g = tempfile.NamedTemporaryFile(mode='w', dir=tempdir, delete=False)
+ try:
+ towrite = section + "\n"
+ towrite += param + " = " + opposite_value
+ g.write(towrite)
+ finally:
+ g.close()
+
+ p = subprocess.Popen(cmdline + ["-s", g.name],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=topdir).communicate()
+ os.unlink(g.name)
+
+ # testparm doesn't display a value if they are equivalent
+ if (value_to_use.lower() != opposite_value.lower()):
+ for line in p[0].decode().splitlines():
+ if not line.strip().startswith(param):
+ return None
+
+ value_found = line.split("=")[1].upper().strip()
+ if value_found != value_to_use.upper():
+ # currently no way to distinguish command lists
+ if param_type == 'list':
+ if ", ".join(value_found.split()) == value_to_use.upper():
+ return None
+
+ # currently no way to identify octal
+ if param_type == 'integer':
+ try:
+ if int(value_to_use, 8) == int(value_found, 8):
+ continue
+ except:
+ pass
+
+ return param, value_to_use, value_found
+
+ return None
def get_documented_parameters(sourcedir):
path = os.path.join(sourcedir, "bin", "default", "docs-xml", "smbdotconf")
@@ -221,34 +312,40 @@ class SmbDotConfTests(TestCase):
failset = set()
- for tuples in self.defaults:
- param, default, context, param_type = tuples
+ with concurrent.futures.ProcessPoolExecutor(max_workers=get_max_worker_count()) as executor:
+ result_futures = []
- if param in self.special_cases:
- continue
- # bad, bad parametric options - we don't have their default values
- if ':' in param:
- continue
- section = None
- if context == "G":
- section = "global"
- elif context == "S":
- section = "test"
- else:
- self.fail("%s has no valid context" % param)
- p = subprocess.Popen(program + ["-s",
- self.smbconf,
- "--section-name",
- section,
- "--parameter-name",
- param],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=self.topdir).communicate()
- result = p[0].decode().upper().strip()
- if result != default.upper():
- if not (result == "" and default == '""'):
- doc_triple = "%s\n Expected: %s" % (param, default)
+ for tuples in self.defaults:
+ param, default, context, param_type = tuples
+
+ if param in self.special_cases:
+ continue
+ # bad, bad parametric options - we don't have their default values
+ if ':' in param:
+ continue
+ section = None
+ if context == "G":
+ section = "global"
+ elif context == "S":
+ section = "test"
+ else:
+ self.fail("%s has no valid context" % param)
+
+ cmdline = program + ["-s",
+ self.smbconf,
+ "--section-name",
+ section,
+ "--parameter-name",
+ param]
+
+ future = executor.submit(check_or_set_smbconf_default, cmdline, self.topdir, param, default)
+ result_futures.append(future)
+
+ for f in concurrent.futures.as_completed(result_futures):
+ if f.result():
+ result, param, default_param = f.result()
+
+ doc_triple = "%s\n Expected: %s" % (param, default_param)
failset.add("%s\n Got: %s" % (doc_triple, result))
if len(failset) > 0:
@@ -262,38 +359,43 @@ class SmbDotConfTests(TestCase):
failset = set()
- for tuples in self.defaults:
- param, default, context, param_type = tuples
+ with concurrent.futures.ProcessPoolExecutor(max_workers=get_max_worker_count()) as executor:
+ result_futures = []
- exceptions = set([
- 'printing',
- 'smbd max async dosmode',
- ])
+ for tuples in self.defaults:
+ param, default, context, param_type = tuples
- if param in exceptions:
- continue
+ exceptions = set([
+ 'printing',
+ 'smbd max async dosmode',
+ ])
+
+ if param in exceptions:
+ continue
+
+ section = None
+ if context == "G":
+ section = "global"
+ elif context == "S":
+ section = "test"
+ else:
+ self.fail("%s has no valid context" % param)
+
+ cmdline = program + ["-s",
+ self.smbconf,
+ "--section-name",
+ section,
+ "--parameter-name",
+ param,
+ "--option",
+ "%s = %s" % (param, default)]
+ future = executor.submit(check_or_set_smbconf_default, cmdline, self.topdir, param, default)
+ result_futures.append(future)
+
+ for f in concurrent.futures.as_completed(result_futures):
+ if f.result():
+ result, param, default_param = f.result()
- section = None
- if context == "G":
- section = "global"
- elif context == "S":
- section = "test"
- else:
- self.fail("%s has no valid context" % param)
- p = subprocess.Popen(program + ["-s",
- self.smbconf,
- "--section-name",
- section,
- "--parameter-name",
- param,
- "--option",
- "%s = %s" % (param, default)],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=self.topdir).communicate()
- result = p[0].decode().upper().strip()
- if result != default.upper():
- if not (result == "" and default == '""'):
doc_triple = "%s\n Expected: %s" % (param, default)
failset.add("%s\n Got: %s" % (doc_triple, result))
@@ -323,105 +425,70 @@ class SmbDotConfTests(TestCase):
failset = set()
- for tuples in self.defaults_all:
- param, default, context, param_type = tuples
+ with concurrent.futures.ProcessPoolExecutor(max_workers=get_max_worker_count()) as executor:
+ result_futures1 = []
+ result_futures2 = []
- if param in ['printing', 'copy', 'include', 'log level']:
- continue
+ for tuples in self.defaults_all:
+ param, default, context, param_type = tuples
- # currently no easy way to set an arbitrary value for these
- if param_type in ['enum', 'boolean-auto']:
- continue
+ if param in ['printing', 'copy', 'include', 'log level']:
+ continue
- if exceptions is not None:
- if param in exceptions:
+ # currently no easy way to set an arbitrary value for these
+ if param_type in ['enum', 'boolean-auto']:
continue
- section = None
- if context == "G":
- section = "global"
- elif context == "S":
- section = "test"
- else:
- self.fail("%s has no valid context" % param)
-
- value_to_use = arbitrary.get(param_type)
- if value_to_use is None:
- self.fail("%s has an invalid type" % param)
-
- p = subprocess.Popen(program + ["-s",
- self.smbconf,
- "--section-name",
- section,
- "--parameter-name",
- param,
- "--option",
- "%s = %s" % (param, value_to_use)],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=self.topdir).communicate()
- result = p[0].decode().upper().strip()
- if result != value_to_use.upper():
- # currently no way to distinguish command lists
- if param_type == 'list':
- if ", ".join(result.split()) == value_to_use.upper():
+ if exceptions is not None:
+ if param in exceptions:
continue
- # currently no way to identify octal
- if param_type == 'integer':
- try:
- if int(value_to_use, 8) == int(p[0].strip(), 8):
- continue
- except:
- pass
+ section = None
+ if context == "G":
+ section = "global"
+ elif context == "S":
+ section = "test"
+ else:
+ self.fail("%s has no valid context" % param)
- doc_triple = "%s\n Expected: %s" % (param, value_to_use)
- failset.add("%s\n Got: %s" % (doc_triple, p[0].upper().strip()))
-
- opposite_value = opposite_arbitrary.get(param_type)
- tempconf = os.path.join(self.tempdir, "tempsmb.conf")
- g = open(tempconf, 'w')
- try:
- towrite = section + "\n"
- towrite += param + " = " + opposite_value
- g.write(towrite)
- finally:
- g.close()
-
- p = subprocess.Popen(program + ["-s",
- tempconf,
- "--suppress-prompt",
- "--option",
- "%s = %s" % (param, value_to_use)],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=self.topdir).communicate()
-
- os.unlink(tempconf)
-
- # testparm doesn't display a value if they are equivalent
- if (value_to_use.lower() != opposite_value.lower()):
- for line in p[0].decode().splitlines():
- if not line.strip().startswith(param):
- continue
+ value_to_use = arbitrary.get(param_type)
+ if value_to_use is None:
+ self.fail("%s has an invalid type" % param)
+
+ cmdline = program + ["-s",
+ self.smbconf,
+ "--section-name",
+ section,
+ "--parameter-name",
+ param,
+ "--option",
+ "%s = %s" % (param, value_to_use)]
+
+ future = executor.submit(set_smbconf_arbitary, cmdline, self.topdir, param, param_type, value_to_use)
+ result_futures1.append(future)
+
+ opposite_value = opposite_arbitrary.get(param_type)
+
+ cmdline = program + ["--suppress-prompt",
+ "--option",
+ "%s = %s" % (param, value_to_use)]
+
+ future = executor.submit(set_smbconf_arbitary_opposite, cmdline, self.topdir, self.tempdir, section, param, opposite_value, value_to_use)
+ result_futures2.append(future)
+
+ for f in concurrent.futures.as_completed(result_futures1):
+ if f.result():
+ result, param, value_to_use = f.result()
+
+ doc_triple = "%s\n Expected: %s" % (param, value_to_use)
+ failset.add("%s\n Got: %s" % (doc_triple, result))
+
+ for f in concurrent.futures.as_completed(result_futures2):
+ if f.result():
+ param, value_to_use, value_found = f.result()
- value_found = line.split("=")[1].upper().strip()
- if value_found != value_to_use.upper():
- # currently no way to distinguish command lists
- if param_type == 'list':
- if ", ".join(value_found.split()) == value_to_use.upper():
- continue
-
- # currently no way to identify octal
- if param_type == 'integer':
- try:
- if int(value_to_use, 8) == int(value_found, 8):
- continue
- except:
- pass
-
- doc_triple = "%s\n Expected: %s" % (param, value_to_use)
- failset.add("%s\n Got: %s" % (doc_triple, value_found))
+ doc_triple = "%s\n Expected: %s" % (param, value_to_use)
+ failset.add("%s\n Got: %s" % (doc_triple, value_found))
if len(failset) > 0:
self.fail(self._format_message(failset,
--
Samba Shared Repository
More information about the samba-cvs
mailing list