[PATCHSET] more python 3

Noel Power nopower at suse.com
Thu Apr 12 10:38:59 UTC 2018


Hi Douglas
On 12/04/18 06:53, Douglas Bagnall wrote:
> Hi Noel,
>
> I was preparing to push this patchset but noticed that one in the middle
> lacks your sign-off:
>
>
>     python/wscript: build grouping library for py2/py3 (--extra-python)
>
>     Grouping library is not used for any delivered libraries, however
>     it is used internally when building local libraries used when
>     "make test TESTS='blah'" is invoked. Failure to provide the grouping
>     library results in missing symbols (and cores) when running tests
oops, sorry about that
>
>
> If you can add a Signed-off-by, you can also add my review.
thanks alot!
please find attached updated patch

Noel
-------------- next part --------------
From 58c5b90c3155024137946d8f29275be012d9ecfb Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Thu, 14 Dec 2017 11:32:23 +0000
Subject: [PATCH 01/31] s4/libcli: python3 port for smb module

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/libcli/pysmb.c       | 45 +++++++++++++++++++++++++++++---------------
 source4/libcli/wscript_build | 16 ++++++++++------
 2 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/source4/libcli/pysmb.c b/source4/libcli/pysmb.c
index 10ab54a9338..a370ab2a98e 100644
--- a/source4/libcli/pysmb.c
+++ b/source4/libcli/pysmb.c
@@ -18,6 +18,7 @@
 */
 
 #include <Python.h>
+#include "python/py3compat.h"
 #include <tevent.h>
 #include <pytalloc.h>
 #include "includes.h"
@@ -115,7 +116,7 @@ static PyObject * py_smb_loadfile(PyObject *self, PyObject *args)
 	status = smb_composite_loadfile(spdata->tree, pytalloc_get_mem_ctx(self), &io);
 	PyErr_NTSTATUS_IS_ERR_RAISE(status);
 
-	return Py_BuildValue("s#", io.out.data, io.out.size);
+	return Py_BuildValue(PYARG_BYTES_LEN, io.out.data, io.out.size);
 }
 
 /*
@@ -125,17 +126,18 @@ static PyObject * py_smb_savefile(PyObject *self, PyObject *args)
 {
 	struct smb_composite_savefile io;
 	const char *filename;
-	char *data;
+	char *data = NULL;
+	Py_ssize_t size = 0;
 	NTSTATUS status;
 	struct smb_private_data *spdata;
 
-	if (!PyArg_ParseTuple(args, "ss:savefile", &filename, &data)) {
+	if (!PyArg_ParseTuple(args, "s"PYARG_BYTES_LEN":savefile", &filename, &data, &size )) {
 		return NULL;
 	}
 
 	io.in.fname = filename;
 	io.in.data = (unsigned char *)data;
-	io.in.size = strlen(data);
+	io.in.size = size;
 
 	spdata = pytalloc_get_ptr(self);
 	status = smb_composite_savefile(spdata->tree, &io);
@@ -157,11 +159,11 @@ static void py_smb_list_callback(struct clilist_file_info *f, const char *mask,
 
 		dict = PyDict_New();
 		if(dict) {
-			PyDict_SetItemString(dict, "name", PyString_FromString(f->name));
+			PyDict_SetItemString(dict, "name", PyStr_FromString(f->name));
 			
 			/* Windows does not always return short_name */
 			if (f->short_name) {
-				PyDict_SetItemString(dict, "short_name", PyString_FromString(f->short_name));
+				PyDict_SetItemString(dict, "short_name", PyStr_FromString(f->short_name));
 			} else {
 				PyDict_SetItemString(dict, "short_name", Py_None);
 			}
@@ -524,11 +526,13 @@ static PyObject *py_close_file(PyObject *self, PyObject *args, PyObject *kwargs)
 
 static PyMethodDef py_smb_methods[] = {
 	{ "loadfile", py_smb_loadfile, METH_VARARGS,
-		"loadfile(path) -> file contents as a string\n\n \
-		Read contents of a file." },
+		"loadfile(path) -> file contents as a "
+		PY_DESC_PY3_BYTES
+		"\n\n Read contents of a file." },
 	{ "savefile", py_smb_savefile, METH_VARARGS,
-		"savefile(path, str) -> None\n\n \
-		Write string str to file." },
+		"savefile(path, str) -> None\n\n Write "
+		PY_DESC_PY3_BYTES
+		" str to file." },
 	{ "list", (PyCFunction)py_smb_list, METH_VARARGS|METH_KEYWORDS,
 		"list(path) -> directory contents as a dictionary\n\n \
 		List contents of a directory. The keys are, \n \
@@ -651,17 +655,27 @@ static PyTypeObject PySMB = {
 
 };
 
-void initsmb(void)
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "smb",
+    .m_doc = "SMB File I/O support",
+    .m_size = -1,
+    .m_methods = NULL,
+};
+
+void initsmb(void);
+
+MODULE_INIT_FUNC(smb)
 {
-	PyObject *m;
+	PyObject *m = NULL;
 
 	if (pytalloc_BaseObject_PyType_Ready(&PySMB) < 0) {
-		return;
+		return m;
 	}
 
-	m = Py_InitModule3("smb", NULL, "SMB File I/O support");
+	m = PyModule_Create(&moduledef);
 	if (m == NULL) {
-	    return;
+	    return m;
 	}
 
 	Py_INCREF(&PySMB);
@@ -685,4 +699,5 @@ void initsmb(void)
 	ADD_FLAGS(FILE_ATTRIBUTE_NONINDEXED);
 	ADD_FLAGS(FILE_ATTRIBUTE_ENCRYPTED);
 	ADD_FLAGS(FILE_ATTRIBUTE_ALL_MASK);
+	return m;
 }
diff --git a/source4/libcli/wscript_build b/source4/libcli/wscript_build
index 38a8f4e0718..220b028cb36 100644
--- a/source4/libcli/wscript_build
+++ b/source4/libcli/wscript_build
@@ -31,12 +31,16 @@ bld.SAMBA_SUBSYSTEM('LIBCLI_SMB_COMPOSITE',
 	private_headers='smb_composite/smb_composite.h',
 	)
 
-bld.SAMBA_PYTHON('pysmb',
-    source='pysmb.c',
-    deps='LIBCLI_SMB_COMPOSITE LIBCLI_SMB2 tevent-util pyparam_util pytalloc-util',
-	public_deps='cli_composite samba-credentials gensec LIBCLI_RESOLVE tevent param_options',
-    realname='samba/smb.so'
-    )
+
+for env in bld.gen_python_environments():
+	pytalloc_util = bld.pyembed_libname('pytalloc-util')
+	pyparam_util =  bld.pyembed_libname('pyparam_util')
+	bld.SAMBA_PYTHON('pysmb',
+		source='pysmb.c',
+		deps='LIBCLI_SMB_COMPOSITE LIBCLI_SMB2 tevent-util %s %s' % (pytalloc_util, pyparam_util),
+		public_deps='cli_composite samba-credentials gensec LIBCLI_RESOLVE tevent param_options',
+		realname='samba/smb.so'
+	)
 
 bld.SAMBA_SUBSYSTEM('LIBCLI_DGRAM',
 	source='dgram/dgramsocket.c dgram/mailslot.c dgram/netlogon.c dgram/browse.c',
-- 
2.13.6


From 2a6730c160b8a8d6d2f808138595003b949b1269 Mon Sep 17 00:00:00 2001
From: David Mulder <dmulder at suse.com>
Date: Fri, 9 Feb 2018 08:42:18 -0700
Subject: [PATCH 02/31] python: create test for pysmb module.

Signed-off-by: David Mulder <dmulder at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/smb.py | 89 +++++++++++++++++++++++++++++++++++++++++++++++
 source4/selftest/tests.py |  1 +
 2 files changed, 90 insertions(+)
 create mode 100644 python/samba/tests/smb.py

diff --git a/python/samba/tests/smb.py b/python/samba/tests/smb.py
new file mode 100644
index 00000000000..4df83233357
--- /dev/null
+++ b/python/samba/tests/smb.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Unix SMB/CIFS implementation. Tests for smb manipulation
+# Copyright (C) David Mulder <dmulder at suse.com> 2018
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import samba, os, random, sys
+from samba import smb
+
+PY3 = sys.version_info[0] == 3
+addom = 'addom.samba.example.com/'
+test_contents = 'abcd'*256
+utf_contents = u'Süßigkeiten Äpfel '*128
+test_literal_bytes_embed_nulls = b'\xff\xfe\x14\x61\x00\x00\x62\x63\x64'*256
+binary_contents = b'\xff\xfe'
+binary_contents = binary_contents + "Hello cruel world of python3".encode('utf8')*128
+test_dir = os.path.join(addom, 'testing_%d' % random.randint(0,0xFFFF))
+test_file = os.path.join(test_dir, 'testing').replace('/', '\\')
+
+class SMBTests(samba.tests.TestCase):
+    def setUp(self):
+        super(SMBTests, self).setUp()
+        self.server = os.environ["SERVER"]
+        creds = self.insta_creds(template=self.get_credentials())
+        self.conn = smb.SMB(self.server,
+                            "sysvol",
+                            lp=self.get_loadparm(),
+                            creds=creds)
+        self.conn.mkdir(test_dir)
+
+    def tearDown(self):
+        super(SMBTests, self).tearDown()
+        try:
+            self.conn.deltree(test_dir)
+        except:
+            pass
+
+    def test_list(self):
+        ls = [f['name'] for f in self.conn.list(addom)]
+        self.assertIn('scripts', ls,
+            msg='"scripts" directory not found in sysvol')
+        self.assertIn('Policies',ls,
+            msg='"Policies" directory not found in sysvol')
+
+    def test_save_load_text(self):
+
+        self.conn.savefile(test_file, test_contents.encode('utf8'))
+
+        contents = self.conn.loadfile(test_file)
+        self.assertEquals(contents.decode('utf8'), test_contents,
+            msg='contents of test file did not match what was written')
+
+    # with python2 this will save/load str type (with embedded nulls)
+    # with python3 this will save/load bytes type
+    def test_save_load_string_bytes(self):
+        self.conn.savefile(test_file, test_literal_bytes_embed_nulls)
+
+        contents = self.conn.loadfile(test_file)
+        self.assertEquals(contents, test_literal_bytes_embed_nulls,
+            msg='contents of test file did not match what was written')
+
+    # python3 only this will save/load unicode
+    def test_save_load_utfcontents(self):
+        if PY3:
+            self.conn.savefile(test_file, utf_contents.encode('utf8'))
+
+            contents = self.conn.loadfile(test_file)
+            self.assertEquals(contents.decode('utf8'), utf_contents,
+                msg='contents of test file did not match what was written')
+
+    # with python2 this will save/load str type
+    # with python3 this will save/load bytes type
+    def test_save_binary_contents(self):
+        self.conn.savefile(test_file, binary_contents);
+
+        contents = self.conn.loadfile(test_file)
+        self.assertEquals(contents, binary_contents,
+            msg='contents of test file did not match what was written')
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 1ab1fff7c16..92231cbd259 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -624,6 +624,7 @@ planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.rpcecho")
 planoldpythontestsuite("nt4_dc", "samba.tests.netbios", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 planoldpythontestsuite("ad_dc:local", "samba.tests.gpo", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 planoldpythontestsuite("ad_dc:local", "samba.tests.dckeytab", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
+planoldpythontestsuite("ad_dc:local", "samba.tests.smb", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 
 planoldpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.registry", extra_args=['-U"$USERNAME%$PASSWORD"'])
 planoldpythontestsuite("ad_dc_ntvfs", "samba.tests.dcerpc.dnsserver", extra_args=['-U"$USERNAME%$PASSWORD"'])
-- 
2.13.6


From 3b6b0b99b04be10eea7a730e60c3e0bd297e7002 Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Wed, 4 Apr 2018 09:53:14 +0100
Subject: [PATCH 03/31] wscript_build: make sure we link extra-python versions
 of libraries

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 auth/credentials/wscript_build      |  5 +++-
 lib/ldb-samba/wscript_build         |  5 +++-
 lib/talloc/wscript                  |  2 +-
 libgpo/wscript_build                |  5 ++--
 source3/wscript_build               |  5 +++-
 source4/auth/wscript_build          |  5 +++-
 source4/lib/messaging/wscript_build |  2 +-
 source4/libnet/wscript_build        |  2 +-
 source4/librpc/wscript_build        | 49 ++++++++++++++++++++-----------------
 source4/param/wscript_build         |  3 ++-
 10 files changed, 50 insertions(+), 33 deletions(-)

diff --git a/auth/credentials/wscript_build b/auth/credentials/wscript_build
index 30c0f727832..73e8e67947a 100644
--- a/auth/credentials/wscript_build
+++ b/auth/credentials/wscript_build
@@ -25,8 +25,11 @@ bld.SAMBA_SUBSYSTEM('CREDENTIALS_NTLM',
 	deps='samba-credentials')
 
 for env in bld.gen_python_environments():
+        pytalloc_util = bld.pyembed_libname('pytalloc-util')
+        pyparam_util = bld.pyembed_libname('pyparam_util')
+
 	bld.SAMBA_PYTHON('pycredentials',
 		source='pycredentials.c',
-		public_deps='samba-credentials cmdline-credentials pytalloc-util pyparam_util CREDENTIALS_KRB5 CREDENTIALS_SECRETS',
+		public_deps='samba-credentials cmdline-credentials %s %s CREDENTIALS_KRB5 CREDENTIALS_SECRETS' % (pytalloc_util, pyparam_util),
 		realname='samba/credentials.so'
 		)
diff --git a/lib/ldb-samba/wscript_build b/lib/ldb-samba/wscript_build
index c538b5a1260..c8464261d61 100644
--- a/lib/ldb-samba/wscript_build
+++ b/lib/ldb-samba/wscript_build
@@ -21,8 +21,11 @@ bld.SAMBA_SUBSYSTEM('ldbwrap',
 for env in bld.gen_python_environments():
     pyparam_util = bld.pyembed_libname('pyparam_util')
     pyldb_util = bld.pyembed_libname('pyldb-util')
+    pyauth = 'pyauth'
+    if bld.env['IS_EXTRA_PYTHON']:
+        pyauth = 'extra-' + pyauth
     bld.SAMBA_PYTHON('python_samba__ldb', 'pyldb.c',
-                     deps='ldbsamba %s ldbwrap %s pyauth' % (pyparam_util, pyldb_util),
+                     deps='ldbsamba %s ldbwrap %s %s' % (pyparam_util, pyldb_util, pyauth),
                      realname='samba/_ldb.so')
 
 bld.SAMBA_MODULE('ldbsamba_extensions',
diff --git a/lib/talloc/wscript b/lib/talloc/wscript
index 083434180bd..1b3167228fe 100644
--- a/lib/talloc/wscript
+++ b/lib/talloc/wscript
@@ -165,7 +165,7 @@ def build(bld):
 
             bld.SAMBA_PYTHON('test_pytalloc',
                             'test_pytalloc.c',
-                            deps='pytalloc',
+                            deps=name,
                             enabled=bld.PYTHON_BUILD_IS_ENABLED(),
                             realname='_test_pytalloc.so',
                             install=False)
diff --git a/libgpo/wscript_build b/libgpo/wscript_build
index f456d417d8a..9fbff9d0974 100644
--- a/libgpo/wscript_build
+++ b/libgpo/wscript_build
@@ -8,7 +8,8 @@ bld.SAMBA3_LIBRARY('gpext',
                    private_library=True)
 
 for env in bld.gen_python_environments():
+    pyparam_util = bld.pyembed_libname('pyparam_util')
+    pyrpc_util = bld.pyembed_libname('pyrpc_util')
     bld.SAMBA3_PYTHON('python_samba_libgpo', 'pygpo.c',
-                     deps='''pyparam_util gpext talloc ads TOKEN_UTIL
-                     auth pyrpc_util''',
+                     deps='%s gpext talloc ads TOKEN_UTIL auth %s' % (pyparam_util, pyrpc_util),
                      realname='samba/gpo.so')
diff --git a/source3/wscript_build b/source3/wscript_build
index cca0f4e82a7..03d5724ee11 100644
--- a/source3/wscript_build
+++ b/source3/wscript_build
@@ -1306,9 +1306,12 @@ for env in bld.gen_python_environments():
                   )
 
 for env in bld.gen_python_environments():
+    pycredentials = 'pycredentials'
+    if bld.env['IS_EXTRA_PYTHON']:
+        pycredentials = 'extra-' + pycredentials
     bld.SAMBA3_PYTHON('pylibsmb',
                   source='libsmb/pylibsmb.c',
-                  deps='smbclient samba-credentials pycredentials',
+                  deps='smbclient samba-credentials %s' % pycredentials,
                   realname='samba/samba3/libsmb_samba_internal.so'
                   )
 
diff --git a/source4/auth/wscript_build b/source4/auth/wscript_build
index 5786fa5a11a..f7508619cd7 100644
--- a/source4/auth/wscript_build
+++ b/source4/auth/wscript_build
@@ -47,10 +47,13 @@ for env in bld.gen_python_environments():
 	pytalloc_util = bld.pyembed_libname('pytalloc-util')
 	pyparam_util = bld.pyembed_libname('pyparam_util')
 	pyldb_util = bld.pyembed_libname('pyldb-util')
+	pycredentials = 'pycredentials'
+        if bld.env['IS_EXTRA_PYTHON']:
+            pycredentials = 'extra-' + pycredentials
 	bld.SAMBA_PYTHON('pyauth',
 		source='pyauth.c',
 		public_deps='auth_system_session',
-		deps='samdb %s %s %s pycredentials auth4' % (pytalloc_util, pyparam_util, pyldb_util),
+		deps='samdb %s %s %s %s auth4' % (pytalloc_util, pyparam_util, pyldb_util, pycredentials),
 		realname='samba/auth.so'
 		)
 
diff --git a/source4/lib/messaging/wscript_build b/source4/lib/messaging/wscript_build
index 4929ccdf515..92f231a62b6 100644
--- a/source4/lib/messaging/wscript_build
+++ b/source4/lib/messaging/wscript_build
@@ -20,6 +20,6 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_messaging',
 		source='pymessaging.c',
-		deps='MESSAGING events pyparam_util pytalloc-util',
+		deps='MESSAGING events %s %s' % (pyparam_util, pytalloc_util),
 		realname='samba/messaging.so'
 		)
diff --git a/source4/libnet/wscript_build b/source4/libnet/wscript_build
index df0a097e520..a4494310f66 100644
--- a/source4/libnet/wscript_build
+++ b/source4/libnet/wscript_build
@@ -20,7 +20,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_dckeytab',
 		source='py_net_dckeytab.c libnet_export_keytab.c',
-		deps='pyrpc_util db-glue krb5 com_err',
+		deps='%s db-glue krb5 com_err' % (pyrpc_util),
 		realname='samba/dckeytab.so',
 		enabled=bld.CONFIG_SET('AD_DC_BUILD_IS_ENABLED')
 		)
diff --git a/source4/librpc/wscript_build b/source4/librpc/wscript_build
index ce893a217c0..b7fd14134f7 100644
--- a/source4/librpc/wscript_build
+++ b/source4/librpc/wscript_build
@@ -267,28 +267,28 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_srvsvc',
 	    source='../../librpc/gen_ndr/py_srvsvc.c',
-	    deps='RPC_NDR_SRVSVC pytalloc-util pyrpc_util',
+	    deps='RPC_NDR_SRVSVC %s %s' % (pytalloc_util, pyrpc_util),
 	    realname='samba/dcerpc/srvsvc.so',
             cflags_end=gen_cflags
 	    )
 
 	bld.SAMBA_PYTHON('python_echo',
 		source='../../librpc/gen_ndr/py_echo.c',
-		deps='RPC_NDR_ECHO pytalloc-util pyrpc_util',
+		deps='RPC_NDR_ECHO %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/echo.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dns',
 		source='../../librpc/gen_ndr/py_dns.c',
-		deps='NDR_DNS pytalloc-util pyrpc_util',
+		deps='NDR_DNS %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/dns.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_winreg',
 		source='../../librpc/gen_ndr/py_winreg.c',
-		deps='RPC_NDR_WINREG pytalloc-util pyrpc_util',
+		deps='RPC_NDR_WINREG %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/winreg.so',
                 cflags_end=gen_cflags
 		)
@@ -296,7 +296,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_initshutdown',
 		source='../../librpc/gen_ndr/py_initshutdown.c',
-		deps='RPC_NDR_INITSHUTDOWN pytalloc-util pyrpc_util',
+		deps='RPC_NDR_INITSHUTDOWN %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/initshutdown.so',
                 cflags_end=gen_cflags
 		)
@@ -304,7 +304,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_epmapper',
 		source='../../librpc/gen_ndr/py_epmapper.c',
-		deps='dcerpc pytalloc-util pyrpc_util',
+		deps='dcerpc %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/epmapper.so',
                 cflags_end=gen_cflags
 		)
@@ -312,7 +312,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_mgmt',
 		source='../../librpc/gen_ndr/py_mgmt.c',
-		deps='pytalloc-util dcerpc pyrpc_util',
+		deps='dcerpc %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/mgmt.so',
                 cflags_end=gen_cflags
 		)
@@ -320,7 +320,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_atsvc',
 		source='../../librpc/gen_ndr/py_atsvc.c',
-		deps='RPC_NDR_ATSVC pytalloc-util pyrpc_util',
+		deps='RPC_NDR_ATSVC %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/atsvc.so',
                 cflags_end=gen_cflags
 		)
@@ -328,7 +328,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_svcctl',
 		source='../../librpc/gen_ndr/py_svcctl.c',
-		deps='RPC_NDR_SVCCTL pytalloc-util pyrpc_util',
+		deps='RPC_NDR_SVCCTL %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/svcctl.so',
                 cflags_end=gen_cflags
 		)
@@ -336,7 +336,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_wkssvc',
 		source='../../librpc/gen_ndr/py_wkssvc.c',
-		deps='RPC_NDR_WKSSVC pytalloc-util pyrpc_util',
+		deps='RPC_NDR_WKSSVC %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/wkssvc.so',
                 cflags_end=gen_cflags
 		)
@@ -344,21 +344,21 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_dfs',
 		source='../../librpc/gen_ndr/py_dfs.c',
-		deps='RPC_NDR_DFS pytalloc-util pyrpc_util',
+		deps='RPC_NDR_DFS %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/dfs.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dcerpc_dcerpc',
 		source='../../librpc/gen_ndr/py_dcerpc.c',
-		deps='NDR_DCERPC pytalloc-util pyrpc_util',
+		deps='NDR_DCERPC %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/dcerpc.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_unixinfo',
 		source='../../librpc/gen_ndr/py_unixinfo.c',
-		deps='RPC_NDR_UNIXINFO pytalloc-util pyrpc_util',
+		deps='RPC_NDR_UNIXINFO %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/unixinfo.so',
                 cflags_end=gen_cflags
 		)
@@ -373,21 +373,24 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_server_id',
 		source='../../librpc/gen_ndr/py_server_id.c',
-		deps='RPC_NDR_SERVER_ID pytalloc-util pyrpc_util',
+		deps='RPC_NDR_SERVER_ID %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/server_id.so',
                 cflags_end=gen_cflags
 		)
 
+	python_netlogon = 'python_netlogon'
+	if bld.env['IS_EXTRA_PYTHON']:
+		python_netlogon = 'extra-' + python_netlogon
 	bld.SAMBA_PYTHON('python_winbind',
 		source='../../librpc/gen_ndr/py_winbind.c',
-		deps='RPC_NDR_WINBIND pytalloc-util pyrpc_util python_netlogon',
+		deps='RPC_NDR_WINBIND %s %s %s' % (pytalloc_util, pyrpc_util, python_netlogon),
 		realname='samba/dcerpc/winbind.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_idmap',
 		source='../../librpc/gen_ndr/py_idmap.c',
-		deps='NDR_IDMAP pytalloc-util pyrpc_util',
+		deps='NDR_IDMAP %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/idmap.so',
                 cflags_end=gen_cflags
 		)
@@ -395,14 +398,14 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_drsuapi',
 		source='../../librpc/gen_ndr/py_drsuapi.c',
-		deps='RPC_NDR_DRSUAPI pytalloc-util pyrpc_util',
+		deps='RPC_NDR_DRSUAPI %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/drsuapi.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dcerpc_dnsp',
 		source='../../librpc/gen_ndr/py_dnsp.c',
-		deps='pytalloc-util pyrpc_util NDR_SECURITY NDR_DNSP',
+		deps='%s %s NDR_SECURITY NDR_DNSP' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/dnsp.so',
                 cflags_end=gen_cflags
 		)
@@ -410,35 +413,35 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_PYTHON('python_dcerpc_xattr',
 		source='../../librpc/gen_ndr/py_xattr.c',
-		deps='pytalloc-util pyrpc_util RPC_NDR_XATTR',
+		deps='%s %s RPC_NDR_XATTR' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/xattr.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dcerpc_idmap',
 		source='../../librpc/gen_ndr/py_idmap.c',
-		deps='pytalloc-util pyrpc_util RPC_NDR_XATTR',
+		deps='%s %s RPC_NDR_XATTR' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/idmap.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dnsserver',
 		source='../../librpc/gen_ndr/py_dnsserver.c',
-		deps='RPC_NDR_DNSSERVER pytalloc-util pyrpc_util',
+		deps='RPC_NDR_DNSSERVER %s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/dnsserver.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('python_dcerpc_smb_acl',
 		source='../../librpc/gen_ndr/py_smb_acl.c',
-		deps='pytalloc-util pyrpc_util',
+		deps='%s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/smb_acl.so',
                 cflags_end=gen_cflags
 		)
 
 	bld.SAMBA_PYTHON('dcerpc_python_messaging',
 		source='../../librpc/gen_ndr/py_messaging.c',
-		deps='pytalloc-util pyrpc_util',
+		deps='%s %s' % (pytalloc_util, pyrpc_util),
 		realname='samba/dcerpc/messaging.so',
                 cflags_end=gen_cflags
 		)
diff --git a/source4/param/wscript_build b/source4/param/wscript_build
index 4a2b3b10be2..1066da4af51 100644
--- a/source4/param/wscript_build
+++ b/source4/param/wscript_build
@@ -39,6 +39,7 @@ bld.SAMBA_SUBSYSTEM('SECRETS',
 for env in bld.gen_python_environments():
 	pytalloc_util = bld.pyembed_libname('pytalloc-util')
 	pyparam_util = bld.pyembed_libname('pyparam_util')
+	libpython = bld.pyembed_libname('LIBPYTHON')
 
 	bld.SAMBA_PYTHON('pyparam',
 	    source='pyparam.c',
@@ -48,7 +49,7 @@ for env in bld.gen_python_environments():
 
 	bld.SAMBA_SUBSYSTEM(pyparam_util,
 	    source='pyparam_util.c',
-	    deps='LIBPYTHON samba-hostconfig %s' % pytalloc_util,
+	    deps='%s samba-hostconfig %s' % (libpython, pytalloc_util),
 	    pyext=True,
 	    enabled=bld.PYTHON_BUILD_IS_ENABLED()
 	    )
-- 
2.13.6


From 23775230cdd7f03f490758a5d943e88ddb05e8ac Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Thu, 5 Apr 2018 16:34:12 +0100
Subject: [PATCH 04/31] s3/param/wscript: build PROVISION subsytem for
 extra-python/py3

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/param/wscript_build | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/source4/param/wscript_build b/source4/param/wscript_build
index 1066da4af51..cf2b7009ea1 100644
--- a/source4/param/wscript_build
+++ b/source4/param/wscript_build
@@ -1,10 +1,16 @@
 #!/usr/bin/env python
-
-bld.SAMBA_SUBSYSTEM('PROVISION',
-	source='provision.c pyparam.c',
-	deps='LIBPYTHON pyparam_util ldb pytalloc-util pyldb-util',
-	pyext=True,
-	enabled=bld.PYTHON_BUILD_IS_ENABLED(),
+for env in bld.gen_python_environments():
+	name = bld.pyembed_libname('PROVISION')
+	pytalloc_util = bld.pyembed_libname('pytalloc-util')
+	pyparam_util = bld.pyembed_libname('pyparam_util')
+	libpython = bld.pyembed_libname('LIBPYTHON')
+	
+	pyldb_util = bld.pyembed_libname('pyldb-util')
+	bld.SAMBA_SUBSYSTEM(name,
+		source='provision.c pyparam.c',
+		deps='%s %s ldb %s %s' % (libpython, pyparam_util, pytalloc_util, pyldb_util),
+		pyext=True,
+		enabled=bld.PYTHON_BUILD_IS_ENABLED(),
 	)
 
 
-- 
2.13.6


From 1427333d184f1b807f35366eea12e1c5d81e4b7e Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Thu, 5 Apr 2018 16:58:33 +0100
Subject: [PATCH 05/31] s3/libnet/wscript: build samba-net lib for
 extra-python/py3

python_net needs to link appropriate samba-net library for extra-py/py3
version so we need to build it for extra-python/py3

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/libnet/wscript_build | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/source4/libnet/wscript_build b/source4/libnet/wscript_build
index a4494310f66..275b35420a7 100644
--- a/source4/libnet/wscript_build
+++ b/source4/libnet/wscript_build
@@ -1,20 +1,25 @@
 #!/usr/bin/env python
 
-bld.SAMBA_LIBRARY('samba-net',
-	source='libnet.c libnet_passwd.c libnet_time.c libnet_rpc.c libnet_join.c libnet_site.c libnet_become_dc.c libnet_unbecome_dc.c libnet_vampire.c libnet_samdump.c libnet_samsync_ldb.c libnet_user.c libnet_group.c libnet_share.c libnet_lookup.c libnet_domain.c userinfo.c groupinfo.c userman.c groupman.c prereq_domain.c libnet_samsync.c',
-	autoproto='libnet_proto.h',
-	public_deps='samba-credentials dcerpc dcerpc-samr RPC_NDR_LSA RPC_NDR_SRVSVC RPC_NDR_DRSUAPI cli_composite LIBCLI_RESOLVE LIBCLI_FINDDCS cli_cldap LIBCLI_FINDDCS gensec_schannel LIBCLI_AUTH ndr smbpasswdparser PROVISION LIBCLI_SAMSYNC LIBTSOCKET',
-	private_library=True,
-	enabled=bld.PYTHON_BUILD_IS_ENABLED()
-	)
-
-
 for env in bld.gen_python_environments():
 	pytalloc_util = bld.pyembed_libname('pytalloc-util')
 	pyrpc_util = bld.pyembed_libname('pyrpc_util')
+	provision = bld.pyembed_libname('PROVISION')
+	name = bld.pyembed_libname('samba-net')
+	auto_proto='libnet_proto.h'
+	if bld.env['IS_EXTRA_PYTHON']:
+		auto_proto=None
+	bld.SAMBA_LIBRARY(name,
+		source='libnet.c libnet_passwd.c libnet_time.c libnet_rpc.c libnet_join.c libnet_site.c libnet_become_dc.c libnet_unbecome_dc.c libnet_vampire.c libnet_samdump.c libnet_samsync_ldb.c libnet_user.c libnet_group.c libnet_share.c libnet_lookup.c libnet_domain.c userinfo.c groupinfo.c userman.c groupman.c prereq_domain.c libnet_samsync.c',
+		autoproto=auto_proto,
+		public_deps='samba-credentials dcerpc dcerpc-samr RPC_NDR_LSA RPC_NDR_SRVSVC RPC_NDR_DRSUAPI cli_composite LIBCLI_RESOLVE LIBCLI_FINDDCS cli_cldap LIBCLI_FINDDCS gensec_schannel LIBCLI_AUTH ndr smbpasswdparser %s LIBCLI_SAMSYNC LIBTSOCKET' % (provision),
+		private_library=True,
+		pyembed=True,
+		enabled=bld.PYTHON_BUILD_IS_ENABLED()
+		)
+
 	bld.SAMBA_PYTHON('python_net',
 		source='py_net.c',
-		deps='samba-net %s %s' % (pyrpc_util, pytalloc_util),
+		deps='%s %s %s' % (name, pyrpc_util, pytalloc_util),
 		realname='samba/net.so'
 		)
 
-- 
2.13.6


From 8a077ff7923d0923fa18216f98a4edbe5b878ee0 Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Thu, 5 Apr 2018 17:05:10 +0100
Subject: [PATCH 06/31] s3/lib/policy/wscript_build: build samba_policy lib for
 extra-python/py3

pypolicy module needs appropriate samba_policy library for
extra-python/py3 therefore we need to build it for it to be available

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/lib/policy/wscript_build | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/source4/lib/policy/wscript_build b/source4/lib/policy/wscript_build
index 2c5abb13154..fa270887bd4 100644
--- a/source4/lib/policy/wscript_build
+++ b/source4/lib/policy/wscript_build
@@ -1,20 +1,23 @@
 #!/usr/bin/env python
 
-bld.SAMBA_LIBRARY('samba-policy',
+
+
+for env in bld.gen_python_environments():
+    pytalloc_util = bld.pyembed_libname('pytalloc-util')
+    samba_policy = bld.pyembed_libname('samba-policy')
+    samba_net = bld.pyembed_libname('samba-net')
+    bld.SAMBA_LIBRARY(samba_policy,
 	source='gp_ldap.c gp_filesys.c gp_manage.c gp_ini.c',
 	pc_files='samba-policy.pc',
-	public_deps='ldb samba-net',
+	public_deps='ldb %s' % samba_net,
 	vnum='0.0.1',
 	pyembed=True,
 	public_headers='policy.h',
 	enabled=bld.PYTHON_BUILD_IS_ENABLED()
 	)
-
-for env in bld.gen_python_environments():
-    pytalloc_util = bld.pyembed_libname('pytalloc-util')
     bld.SAMBA_PYTHON(
         'py_policy',
         source='pypolicy.c',
-        public_deps=' '.join(['samba-policy', pytalloc_util]),
+        public_deps='%s %s' % (samba_policy, pytalloc_util),
         realname='samba/policy.so'
     )
-- 
2.13.6


From 7d3d2d6ddc3ccaac2dd41164e62b306b50504f09 Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power at suse.com>
Date: Fri, 6 Apr 2018 16:35:49 +0100
Subject: [PATCH 07/31] python/wscript: build grouping library for py2/py3
 (--extra-python)

Grouping library is not used for any delivered libraries, however
it is used internally when building local libraries used when
"make test TESTS='blah'" is invoked. Failure to provide the grouping
library results in missing symbols (and cores) when running tests

Signed-off-by: Noel Power <noel.power at suse.com>
Reviewed-by: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
---
 python/wscript | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/python/wscript b/python/wscript
index 211fac4de62..2978f187b67 100644
--- a/python/wscript
+++ b/python/wscript
@@ -39,23 +39,21 @@ def configure(conf):
         f.close()
 
 def build(bld):
-    bld.SAMBA_LIBRARY('samba_python',
-                      source=[],
-                      deps='''
-                           LIBPYTHON
-                           pytalloc-util
-                           pyrpc_util
-                           ''',
-                      grouping_library=True,
-                      private_library=True,
-                      pyembed=True,
-                      enabled=bld.PYTHON_BUILD_IS_ENABLED())
+
 
     for env in bld.gen_python_environments():
         pytalloc_util = bld.pyembed_libname('pytalloc-util')
         pyparam_util = bld.pyembed_libname('pyparam_util')
         libpython = bld.pyembed_libname('LIBPYTHON')
-
+        pyrpc_util = bld.pyembed_libname('pyrpc_util')
+        samba_python = bld.pyembed_libname('samba_python')
+        bld.SAMBA_LIBRARY(samba_python,
+                      source=[],
+                      deps='%s %s %s' % (libpython, pytalloc_util, pyrpc_util),
+                      grouping_library=True,
+                      private_library=True,
+                      pyembed=True,
+                      enabled=bld.PYTHON_BUILD_IS_ENABLED())
         bld.SAMBA_PYTHON('python_glue',
                          source='pyglue.c',
                          deps='''
-- 
2.13.6


From 9daf3ce9cb26683744cb39991efd20af82c503b6 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 10:32:06 +1200
Subject: [PATCH 08/31] python: bulk replace dict.iteritems to items for py3

In py3, iterxxx methods are removed.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 buildtools/wafsamba/samba_version.py           |  2 +-
 lib/tdb/python/tdbdump.py                      |  2 +-
 python/samba/emulate/traffic.py                | 16 ++++++++--------
 python/samba/graph.py                          | 10 +++++-----
 python/samba/netcmd/__init__.py                |  2 +-
 python/samba/netcmd/main.py                    |  4 +---
 python/samba/netcmd/visualize.py               | 26 +++++++++++++-------------
 python/samba/provision/__init__.py             |  4 ++--
 python/samba/tests/netcmd.py                   |  5 ++---
 selftest/subunithelper.py                      |  2 +-
 source4/dsdb/tests/python/linked_attributes.py |  4 ++--
 source4/selftest/tests.py                      |  2 +-
 12 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/buildtools/wafsamba/samba_version.py b/buildtools/wafsamba/samba_version.py
index 950a855c1ef..be26439f1a6 100644
--- a/buildtools/wafsamba/samba_version.py
+++ b/buildtools/wafsamba/samba_version.py
@@ -95,7 +95,7 @@ also accepted as dictionary entries here
         self.VENDOR_SUFFIX=None
         self.VENDOR_PATCH=None
 
-        for a, b in version_dict.iteritems():
+        for a, b in version_dict.items():
             if a.startswith("SAMBA_VERSION_"):
                 setattr(self, a[14:], b)
             else:
diff --git a/lib/tdb/python/tdbdump.py b/lib/tdb/python/tdbdump.py
index a16d26ed523..08769eaa3e5 100644
--- a/lib/tdb/python/tdbdump.py
+++ b/lib/tdb/python/tdbdump.py
@@ -9,5 +9,5 @@ if len(sys.argv) < 2:
     sys.exit(1)
 
 db = tdb.Tdb(sys.argv[1])
-for (k, v) in db.iteritems():
+for (k, v) in db.items():
     print("{\nkey(%d) = %r\ndata(%d) = %r\n}" % (len(k), k, len(v), v))
diff --git a/python/samba/emulate/traffic.py b/python/samba/emulate/traffic.py
index 240d0ebd260..0fb60b4c87b 100644
--- a/python/samba/emulate/traffic.py
+++ b/python/samba/emulate/traffic.py
@@ -1065,7 +1065,7 @@ def guess_server_address(conversations):
 
 def stringify_keys(x):
     y = {}
-    for k, v in x.iteritems():
+    for k, v in x.items():
         k2 = '\t'.join(k)
         y[k2] = v
     return y
@@ -1073,7 +1073,7 @@ def stringify_keys(x):
 
 def unstringify_keys(x):
     y = {}
-    for k, v in x.iteritems():
+    for k, v in x.items():
         t = tuple(str(k).split('\t'))
         y[t] = v
     return y
@@ -1133,12 +1133,12 @@ class TrafficModel(object):
 
     def save(self, f):
         ngrams = {}
-        for k, v in self.ngrams.iteritems():
+        for k, v in self.ngrams.items():
             k = '\t'.join(k)
             ngrams[k] = dict(Counter(v))
 
         query_details = {}
-        for k, v in self.query_details.iteritems():
+        for k, v in self.query_details.items():
             query_details[k] = dict(Counter('\t'.join(x) if x else '-'
                                             for x in v))
 
@@ -1161,15 +1161,15 @@ class TrafficModel(object):
 
         d = json.load(f)
 
-        for k, v in d['ngrams'].iteritems():
+        for k, v in d['ngrams'].items():
             k = tuple(str(k).split('\t'))
             values = self.ngrams.setdefault(k, [])
-            for p, count in v.iteritems():
+            for p, count in v.items():
                 values.extend([str(p)] * count)
 
-        for k, v in d['query_details'].iteritems():
+        for k, v in d['query_details'].items():
             values = self.query_details.setdefault(str(k), [])
-            for p, count in v.iteritems():
+            for p, count in v.items():
                 if p == '-':
                     values.extend([()] * count)
                 else:
diff --git a/python/samba/graph.py b/python/samba/graph.py
index f626287800d..3b8f61194f4 100644
--- a/python/samba/graph.py
+++ b/python/samba/graph.py
@@ -129,7 +129,7 @@ def shorten_vertex_names(edges, vertices, suffix=',...', aggressive=False):
                 break
         else:
             map = dict((k, v.replace(',CN=Servers,', ',**,'))
-                       for k, v in map.iteritems())
+                       for k, v in map.items())
             replacements.append(('**', 'CN=Servers'))
 
         for v in vertices2:
@@ -137,7 +137,7 @@ def shorten_vertex_names(edges, vertices, suffix=',...', aggressive=False):
                 break
         else:
             map = dict((k, v.replace('CN=NTDS Settings,', '*,'))
-                       for k, v in map.iteritems())
+                       for k, v in map.items())
             replacements.append(('*', 'CN=NTDS Settings'))
 
         edges2 = [(map.get(a, a), map.get(b, b)) for a, b in edges2]
@@ -463,11 +463,11 @@ def find_transitive_distance(vertices, edges):
     for i in range(inf):
         changed = False
         new_distances = {}
-        for v, d in distances.iteritems():
+        for v, d in distances.items():
             new_d = d.copy()
             new_distances[v] = new_d
-            for dest, cost in d.iteritems():
-                for leaf, cost2 in distances[dest].iteritems():
+            for dest, cost in d.items():
+                for leaf, cost2 in distances[dest].items():
                     new_cost = cost + cost2
                     old_cost = d.get(leaf, inf)
                     if new_cost < old_cost:
diff --git a/python/samba/netcmd/__init__.py b/python/samba/netcmd/__init__.py
index 9037d033190..77976780150 100644
--- a/python/samba/netcmd/__init__.py
+++ b/python/samba/netcmd/__init__.py
@@ -130,7 +130,7 @@ class Command(object):
             prog=prog,epilog=epilog)
         parser.add_options(self.takes_options)
         optiongroups = {}
-        for name, optiongroup in self.takes_optiongroups.iteritems():
+        for name, optiongroup in self.takes_optiongroups.items():
             optiongroups[name] = optiongroup(parser)
             parser.add_option_group(optiongroups[name])
         return parser, optiongroups
diff --git a/python/samba/netcmd/main.py b/python/samba/netcmd/main.py
index 3b56f1a724e..40762fabdad 100644
--- a/python/samba/netcmd/main.py
+++ b/python/samba/netcmd/main.py
@@ -42,12 +42,10 @@ class cache_loader(dict):
         except KeyError:
             return default
 
-    def iteritems(self):
+    def items(self):
         for key in self:
             yield (key, self[key])
 
-    def items(self):
-        return list(self.iteritems())
 
 class cmd_sambatool(SuperCommand):
     """Main samba administration tool."""
diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py
index 3e5b9501e3a..b983e1891a9 100644
--- a/python/samba/netcmd/visualize.py
+++ b/python/samba/netcmd/visualize.py
@@ -178,7 +178,7 @@ def get_partition_maps(samdb):
     }
 
     long_to_short = {}
-    for s, l in short_to_long.iteritems():
+    for s, l in short_to_long.items():
         long_to_short[l] = s
 
     return short_to_long, long_to_short
@@ -266,18 +266,18 @@ class cmd_reps(GraphCommand):
                 # get_reps_tables() returns two dictionaries mapping
                 # dns to NCReplica objects
                 c, n = remote_dsa.get_rep_tables()
-                for part, rep in c.iteritems():
+                for part, rep in c.items():
                     if partition is None or part == partition:
                         nc_reps[part]['current'].append((dsa_dn, rep))
-                for part, rep in n.iteritems():
+                for part, rep in n.items():
                     if partition is None or part == partition:
                         nc_reps[part]['needed'].append((dsa_dn, rep))
 
         all_edges = {'needed':  {'to': [], 'from': []},
                      'current': {'to': [], 'from': []}}
 
-        for partname, part in nc_reps.iteritems():
-            for state, edgelists in all_edges.iteritems():
+        for partname, part in nc_reps.items():
+            for state, edgelists in all_edges.items():
                 for dsa_dn, rep in part[state]:
                     short_name = long_partitions.get(partname, partname)
                     for r in rep.rep_repsFrom:
@@ -302,12 +302,12 @@ class cmd_reps(GraphCommand):
                 'from': "RepsFrom objects for %s",
                 'to': "RepsTo objects for %s",
             }
-            for state, edgelists in all_edges.iteritems():
-                for direction, items in edgelists.iteritems():
+            for state, edgelists in all_edges.items():
+                for direction, items in edgelists.items():
                     part_edges = defaultdict(list)
                     for src, dest, part in items:
                         part_edges[part].append((src, dest))
-                    for part, edges in part_edges.iteritems():
+                    for part, edges in part_edges.items():
                         s = distance_matrix(None, edges,
                                             utf8=utf8,
                                             colour=color_scheme,
@@ -324,8 +324,8 @@ class cmd_reps(GraphCommand):
         dot_vertices = set()
         used_colours = {}
         key_set = set()
-        for state, edgelist in all_edges.iteritems():
-            for direction, items in edgelist.iteritems():
+        for state, edgelist in all_edges.items():
+            for direction, items in edgelist.items():
                 for src, dest, part in items:
                     colour = used_colours.setdefault((part),
                                                      colour_hash((part,
@@ -472,7 +472,7 @@ class cmd_ntdsconn(GraphCommand):
                 source_denies = []
                 dest_denies = []
                 both_deny = []
-                for e, conn in edges.iteritems():
+                for e, conn in edges.items():
                     if conn.dest_attests:
                         graph_edges.append(e)
                         if not conn.src_attests:
@@ -517,7 +517,7 @@ class cmd_ntdsconn(GraphCommand):
         edge_styles = []
         edge_labels = []
         n_servers = len(dsas)
-        for k, e in sorted(edges.iteritems()):
+        for k, e in sorted(edges.items()):
             dot_edges.append(k)
             if e.observations == n_servers or not talk_to_remote:
                 edge_colours.append('#000000')
@@ -573,6 +573,6 @@ class cmd_visualize(SuperCommand):
     """Produces graphical representations of Samba network state"""
     subcommands = {}
 
-    for k, v in globals().iteritems():
+    for k, v in globals().items():
         if k.startswith('cmd_'):
             subcommands[k[4:]] = v()
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index dcdb2b7bd20..1709e75184e 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -774,11 +774,11 @@ def make_smbconf(smbconf, hostname, domain, realm, targetdir,
     f = open(smbconf, 'w')
     try:
         f.write("[globals]\n")
-        for key, val in global_settings.iteritems():
+        for key, val in global_settings.items():
             f.write("\t%s = %s\n" % (key, val))
         f.write("\n")
 
-        for name, path in shares.iteritems():
+        for name, path in shares.items():
             f.write("[%s]\n" % name)
             f.write("\tpath = %s\n" % path)
             f.write("\tread only = no\n")
diff --git a/python/samba/tests/netcmd.py b/python/samba/tests/netcmd.py
index 54f05da35fa..2867c031a17 100644
--- a/python/samba/tests/netcmd.py
+++ b/python/samba/tests/netcmd.py
@@ -36,14 +36,13 @@ class NetCmdTestCase(samba.tests.TestCase):
         return cmd.outf.getvalue(), cmd.errf.getvalue()
 
     def iter_all_subcommands(self):
-        todo = []
-        todo.extend(cmd_sambatool.subcommands.items())
+        todo = list(cmd_sambatool.subcommands.items())
         while todo:
             (path, cmd) = todo.pop()
             yield path, cmd
             subcmds = getattr(cmd, "subcommands", {})
             todo.extend([(path + " " + k, v) for (k, v) in
-                subcmds.iteritems()])
+                subcmds.items()])
 
 
 class TestParmTests(NetCmdTestCase):
diff --git a/selftest/subunithelper.py b/selftest/subunithelper.py
index 9755f926d4f..f20f3fe8ba7 100644
--- a/selftest/subunithelper.py
+++ b/selftest/subunithelper.py
@@ -262,7 +262,7 @@ def read_test_regexes(*names):
 
 
 def find_in_list(regexes, fullname):
-    for regex, reason in regexes.iteritems():
+    for regex, reason in regexes.items():
         if re.match(regex, fullname):
             if reason is None:
                 return ""
diff --git a/source4/dsdb/tests/python/linked_attributes.py b/source4/dsdb/tests/python/linked_attributes.py
index d40192badc7..8586fb81528 100644
--- a/source4/dsdb/tests/python/linked_attributes.py
+++ b/source4/dsdb/tests/python/linked_attributes.py
@@ -173,7 +173,7 @@ class LATests(samba.tests.TestCase):
         except ldb.LdbError as e:
             (num, msg) = e.args
             if num != errcode:
-                lut = {v: k for k, v in vars(ldb).iteritems()
+                lut = {v: k for k, v in vars(ldb).items()
                        if k.startswith('ERR_') and isinstance(v, int)}
                 self.fail("%s, expected "
                           "LdbError %s, (%d) "
@@ -181,7 +181,7 @@ class LATests(samba.tests.TestCase):
                                            lut.get(errcode), errcode,
                                            lut.get(num), num))
         else:
-            lut = {v: k for k, v in vars(ldb).iteritems()
+            lut = {v: k for k, v in vars(ldb).items()
                    if k.startswith('ERR_') and isinstance(v, int)}
             self.fail("%s, expected "
                       "LdbError %s, (%d) "
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 92231cbd259..f86623ab0b5 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -563,7 +563,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex
     py_path = list(extra_path)
     if py_path:
         environ["PYTHONPATH"] = ":".join(["$PYTHONPATH"] + py_path)
-    args = ["%s=%s" % item for item in environ.iteritems()]
+    args = ["%s=%s" % item for item in environ.items()]
     args += [subunitrun, "$LISTOPT", "$LOADLIST", module]
     args += extra_args
     if name is None:
-- 
2.13.6


From bd8548c447a457dffec0d71d7a33e3213d27774e Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Thu, 12 Apr 2018 16:07:24 +1200
Subject: [PATCH 09/31] python: bulk port tdb iterkeys for py3

In py3, `dict.iterkeys()` is removed, we need to use `keys()` instead.
This is compatible with py2 since `dict.keys()` exists for py2.

tdb pretents to be a dict, however, not completely.
It provides `iterkeys()` for py2 only, and `keys()` for py3 only,
which means replace `iterkeys()` to `keys()` will break py2.

In python, iter a dict will implicitly iter on keys.
Use this feature to work around.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/samba3/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/samba/samba3/__init__.py b/python/samba/samba3/__init__.py
index f7927836c98..b145e432c97 100644
--- a/python/samba/samba3/__init__.py
+++ b/python/samba/samba3/__init__.py
@@ -142,7 +142,7 @@ class IdmapDatabase(DbDatabase):
 
     def ids(self):
         """Retrieve a list of all ids in this database."""
-        for k in self.db.iterkeys():
+        for k in self.db:
             if k.startswith(IDMAP_USER_PREFIX):
                 yield k.rstrip(b"\0").split(b" ")
             if k.startswith(IDMAP_GROUP_PREFIX):
@@ -213,7 +213,7 @@ class SecretsDatabase(DbDatabase):
         return self.db.get("SECRETS/DOMGUID/%s" % host)
 
     def ldap_dns(self):
-        for k in self.db.iterkeys():
+        for k in self.db:
             if k.startswith("SECRETS/LDAP_BIND_PW/"):
                 yield k[len("SECRETS/LDAP_BIND_PW/"):].rstrip("\0")
 
@@ -222,7 +222,7 @@ class SecretsDatabase(DbDatabase):
 
         :return: Iterator over the names of domains in this database.
         """
-        for k in self.db.iterkeys():
+        for k in self.db:
             if k.startswith("SECRETS/SID/"):
                 yield k[len("SECRETS/SID/"):].rstrip("\0")
 
@@ -248,7 +248,7 @@ class SecretsDatabase(DbDatabase):
         return self.db.get("SECRETS/$DOMTRUST.ACC/%s" % host)
 
     def trusted_domains(self):
-        for k in self.db.iterkeys():
+        for k in self.db:
             if k.startswith("SECRETS/$DOMTRUST.ACC/"):
                 yield k[len("SECRETS/$DOMTRUST.ACC/"):].rstrip("\0")
 
-- 
2.13.6


From aed1cef71401cdfedb5cd3f9966468511bb307dc Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 10:44:24 +1200
Subject: [PATCH 10/31] python: bulk replace dict.itervalues to values for py3

In py3, iterxxx methods are removed.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/scripting/bin/w32err_code.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source4/scripting/bin/w32err_code.py b/source4/scripting/bin/w32err_code.py
index 665bfdf1ceb..b05d25785c1 100755
--- a/source4/scripting/bin/w32err_code.py
+++ b/source4/scripting/bin/w32err_code.py
@@ -194,7 +194,7 @@ class WerrorGenerator(object):
         fp.close()
 
     def _lookup_error_by_name(self, err_name, defined_errors):
-        for err in defined_errors.itervalues():
+        for err in defined_errors.values():
             if err['err_name'] == err_name:
                 return err
         return None
@@ -212,7 +212,7 @@ class WerrorGenerator(object):
         new_errors = {}
         diff_code_errors = []
         diff_name_errors = []
-        for err_def in errors.itervalues():
+        for err_def in errors.values():
             add_error = True
             # try get defined error by code
             if defined_errors.has_key(err_def['code']):
-- 
2.13.6


From 9bbc959f91020bc393d3343478b9af3e6085179c Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 16:03:34 +1200
Subject: [PATCH 11/31] python: bulk replace file to open for py3

The builtin function `file` was removed in py3. Use `open` instead.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/netcmd/gpo.py         | 6 +++---
 python/samba/provision/sambadns.py | 2 +-
 python/samba/subunit/run.py        | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/python/samba/netcmd/gpo.py b/python/samba/netcmd/gpo.py
index 5f4ad0ebca0..cd946d9fa72 100644
--- a/python/samba/netcmd/gpo.py
+++ b/python/samba/netcmd/gpo.py
@@ -252,7 +252,7 @@ def copy_directory_remote_to_local(conn, remotedir, localdir):
                 os.mkdir(l_name)
             else:
                 data = conn.loadfile(r_name)
-                file(l_name, 'w').write(data)
+                open(l_name, 'w').write(data)
 
 
 def copy_directory_local_to_remote(conn, localdir, remotedir):
@@ -274,7 +274,7 @@ def copy_directory_local_to_remote(conn, localdir, remotedir):
                 r_dirs.append(r_name)
                 conn.mkdir(r_name)
             else:
-                data = file(l_name, 'r').read()
+                data = open(l_name, 'r').read()
                 conn.savefile(r_name, data)
 
 
@@ -943,7 +943,7 @@ class cmd_create(Command):
             os.mkdir(os.path.join(gpodir, "Machine"))
             os.mkdir(os.path.join(gpodir, "User"))
             gpt_contents = "[General]\r\nVersion=0\r\n"
-            file(os.path.join(gpodir, "GPT.INI"), "w").write(gpt_contents)
+            open(os.path.join(gpodir, "GPT.INI"), "w").write(gpt_contents)
         except Exception as e:
             raise CommandError("Error Creating GPO files", e)
 
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index 7a85546c53e..3d59c701066 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -806,7 +806,7 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
     domainpart_file = os.path.join(dns_dir, partfile[domaindn])
     try:
         os.mkdir(dns_samldb_dir)
-        file(domainpart_file, 'w').close()
+        open(domainpart_file, 'w').close()
 
         # Fill the basedn and @OPTION records in domain partition
         dom_url = "%s://%s" % (backend_store, domainpart_file)
diff --git a/python/samba/subunit/run.py b/python/samba/subunit/run.py
index bb598b42452..8f32d46ef49 100755
--- a/python/samba/subunit/run.py
+++ b/python/samba/subunit/run.py
@@ -79,7 +79,7 @@ class TestProtocolClient(unittest.TestResult):
     suite = make_suite()
     # Create a stream (any object with a 'write' method). This should accept
     # bytes not strings: subunit is a byte orientated protocol.
-    stream = file('tests.log', 'wb')
+    stream = open('tests.log', 'wb')
     # Create a subunit result object which will output to the stream
     result = subunit.TestProtocolClient(stream)
     # Optionally, to get timing data for performance analysis, wrap the
-- 
2.13.6


From 6a60720e520555f8df4a849098da041075bbfe0f Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:49:40 +1200
Subject: [PATCH 12/31] python: bulk convert zip to list

In py3, zip will return a iterator other than a list.
Convert it to a list to support both py2 and py3.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/emulate/traffic.py | 6 +++---
 python/samba/tests/graph.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/samba/emulate/traffic.py b/python/samba/emulate/traffic.py
index 0fb60b4c87b..9d95e3a14d5 100644
--- a/python/samba/emulate/traffic.py
+++ b/python/samba/emulate/traffic.py
@@ -1398,9 +1398,9 @@ def replay(conversations,
         print(("we have %d accounts but %d conversations" %
                (accounts, conversations)), file=sys.stderr)
 
-    cstack = zip(sorted(conversations,
-                        key=lambda x: x.start_time, reverse=True),
-                 accounts)
+    cstack = list(zip(
+        sorted(conversations, key=lambda x: x.start_time, reverse=True),
+        accounts))
 
     # Set the process group so that the calling scripts are not killed
     # when the forked child processes are killed.
diff --git a/python/samba/tests/graph.py b/python/samba/tests/graph.py
index d1824bc3ef9..0a95afb1859 100644
--- a/python/samba/tests/graph.py
+++ b/python/samba/tests/graph.py
@@ -73,7 +73,7 @@ class DotFileTests(samba.tests.TestCaseInTempDir):
     def test_basic_dot_files(self):
         vertices = tuple('abcdefgh')
         all_edges = tuple(itertools.combinations(vertices, 2))
-        line_edges = zip(vertices[1:], vertices[:-1])
+        line_edges = list(zip(vertices[1:], vertices[:-1]))
         ring_edges = line_edges + [(vertices[0], vertices[-1])]
         no_edges = []
         # even join to even numbers, odd to odd
-- 
2.13.6


From 50a1ceaa8fa0f8464bda0512d579253db9c3af4d Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Thu, 12 Apr 2018 10:54:34 +1200
Subject: [PATCH 13/31] python: bulk convert map to list

In py3, map is a iter, not a list.
Convert it to list when necessary.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/dbchecker.py                | 13 +++++++------
 python/samba/tests/docs.py               |  3 +--
 source4/torture/drs/python/getnc_exop.py |  9 +++++----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py
index fb968a56481..63408582a31 100644
--- a/python/samba/dbchecker.py
+++ b/python/samba/dbchecker.py
@@ -1963,13 +1963,14 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         # If we modify the pass-by-reference attrs variable, then we get a
         # replPropertyMetadata for every object that we check.
         attrs = list(attrs)
-        if "dn" in map(str.lower, attrs):
+        attrs_lower = list(map(str.lower, attrs))
+        if "dn" in attrs_lower:
             attrs.append("name")
-        if "distinguishedname" in map(str.lower, attrs):
+        if "distinguishedname" in attrs_lower:
             attrs.append("name")
-        if str(dn.get_rdn_name()).lower() in map(str.lower, attrs):
+        if str(dn.get_rdn_name()).lower() in attrs_lower:
             attrs.append("name")
-        if 'name' in map(str.lower, attrs):
+        if 'name' in attrs_lower:
             attrs.append(dn.get_rdn_name())
             attrs.append("isDeleted")
             attrs.append("systemFlags")
@@ -2247,7 +2248,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             error_count += 1
             self.err_missing_objectclass(dn)
 
-        if ("*" in attrs or "name" in map(str.lower, attrs)):
+        if ("*" in attrs or "name" in attrs_lower):
             if name_val is None:
                 error_count += 1
                 self.report("ERROR: Not fixing missing 'name' on '%s'" % (str(obj.dn)))
@@ -2306,7 +2307,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 self.fix_metadata(obj, att)
 
         if self.is_fsmo_role(dn):
-            if "fSMORoleOwner" not in obj and ("*" in attrs or "fsmoroleowner" in map(str.lower, attrs)):
+            if "fSMORoleOwner" not in obj and ("*" in attrs or "fsmoroleowner" in attrs_lower):
                 self.err_no_fsmoRoleOwner(obj)
                 error_count += 1
 
diff --git a/python/samba/tests/docs.py b/python/samba/tests/docs.py
index 3d896d7c7d2..3b06be8ad83 100644
--- a/python/samba/tests/docs.py
+++ b/python/samba/tests/docs.py
@@ -30,8 +30,7 @@ import xml.etree.ElementTree as ET
 class TestCase(samba.tests.TestCaseInTempDir):
 
     def _format_message(self, parameters, message):
-        parameters = list(parameters)
-        parameters = map(str, parameters)
+        parameters = list(map(str, parameters))
         parameters.sort()
         return message + '\n\n    %s' % ('\n    '.join(parameters))
 
diff --git a/source4/torture/drs/python/getnc_exop.py b/source4/torture/drs/python/getnc_exop.py
index d92a535932b..83d6e86ec35 100644
--- a/source4/torture/drs/python/getnc_exop.py
+++ b/source4/torture/drs/python/getnc_exop.py
@@ -926,15 +926,16 @@ class DrsReplicaPrefixMapTestCase(drs_base.DrsBaseTestCase):
         schi.id_prefix = 0
 
         if 'schemaInfo' in res[0]:
-            schi.oid.length = len(map(ord, str(res[0]['schemaInfo'])))
-            schi.oid.binary_oid = map(ord, str(res[0]['schemaInfo']))
+            oid = list(map(ord, str(res[0]['schemaInfo'])))
         else:
             schema_info = drsblobs.schemaInfoBlob()
             schema_info.revision = 0
             schema_info.marker = 0xFF
             schema_info.invocation_id = misc.GUID(samdb.get_invocation_id())
-            schi.oid.length = len(map(ord, ndr_pack(schema_info)))
-            schi.oid.binary_oid = map(ord, ndr_pack(schema_info))
+            oid = list(map(ord, ndr_pack(schema_info)))
+
+        schi.oid.length = len(oid)
+        schi.oid.binary_oid = oid
 
         pfm.ctr.mappings = pfm.ctr.mappings + [schi]
         pfm.ctr.num_mappings += 1
-- 
2.13.6


From b82762dd2ba545f3d723f5dfda9795478c23273c Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 16:32:19 +1200
Subject: [PATCH 14/31] python: fix unicode escape in doc string

The doc string has `\u` mark inside, which will cause encoding error in
py3. prefix `r` to doc string to fix.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/netcmd/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/samba/netcmd/common.py b/python/samba/netcmd/common.py
index 5c0bd95f089..2d0ff620bae 100644
--- a/python/samba/netcmd/common.py
+++ b/python/samba/netcmd/common.py
@@ -23,7 +23,7 @@ from samba.net import Net
 
 
 def _get_user_realm_domain(user):
-    """ get the realm or the domain and the base user
+    r""" get the realm or the domain and the base user
         from user like:
         * username
         * DOMAIN\username
-- 
2.13.6


From f8c6783615949f19ec463fc24d1eb2ee959531ad Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 16:27:47 +1200
Subject: [PATCH 15/31] selftest: enable py3 for samba.tests.dcerpc.sam

1. Fix invalid declaration syntax for toArray
2. Simplify toArray implementation with list comprehension.
3. Remove ending L for long integer.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/dcerpc/sam.py | 10 ++++------
 source4/selftest/tests.py        |  2 +-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/python/samba/tests/dcerpc/sam.py b/python/samba/tests/dcerpc/sam.py
index 1c48d7e8170..d42c4f797fa 100644
--- a/python/samba/tests/dcerpc/sam.py
+++ b/python/samba/tests/dcerpc/sam.py
@@ -22,12 +22,10 @@
 from samba.dcerpc import samr, security
 from samba.tests import RpcInterfaceTestCase
 
+
 # FIXME: Pidl should be doing this for us
-def toArray((handle, array, num_entries)):
-    ret = []
-    for x in range(num_entries):
-        ret.append((array.entries[x].idx, array.entries[x].name))
-    return ret
+def toArray(handle, array, num_entries):
+    return [(entry.idx, entry.name) for entry in array.entries[:num_entries]]
 
 
 class SamrTests(RpcInterfaceTestCase):
@@ -45,6 +43,6 @@ class SamrTests(RpcInterfaceTestCase):
 
     def test_EnumDomains(self):
         handle = self.conn.Connect2(None, security.SEC_FLAG_MAXIMUM_ALLOWED)
-        domains = toArray(self.conn.EnumDomains(handle, 0, 4294967295L))
+        domains = toArray(*self.conn.EnumDomains(handle, 0, 4294967295))
         self.conn.Close(handle)
 
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index f86623ab0b5..47ff7f24653 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -576,7 +576,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex
 
 planoldpythontestsuite("ad_dc_ntvfs:local", "samba.tests.gensec", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 planoldpythontestsuite("none", "simple", extra_path=["%s/lib/tdb/python/tests" % srcdir()], name="tdb.python")
-planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.sam")
+planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.sam", py3_compatible=True)
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dsdb", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.dsdb_lock")
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare")
-- 
2.13.6


From 2cf5f5a1537c3f0413d581a2a4a94b402f339acc Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 11:39:56 +1200
Subject: [PATCH 16/31] selftest: enable py3 for samba.tests.dcerpc.bare

Prefix b to bytes.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/dcerpc/bare.py | 8 ++++----
 source4/selftest/tests.py         | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/python/samba/tests/dcerpc/bare.py b/python/samba/tests/dcerpc/bare.py
index 270c7af382e..380164cff10 100644
--- a/python/samba/tests/dcerpc/bare.py
+++ b/python/samba/tests/dcerpc/bare.py
@@ -30,7 +30,7 @@ class BareTestCase(samba.tests.TestCase):
         x = ClientConnection("ncalrpc:localhost[DEFAULT]",
                 ("60a15ec5-4de8-11d7-a637-005056a20182", 1),
                 lp_ctx=samba.tests.env_loadparm())
-        self.assertEquals("\x01\x00\x00\x00", x.request(0, chr(0) * 4))
+        self.assertEquals(b"\x01\x00\x00\x00", x.request(0, chr(0) * 4))
 
     def test_two_contexts(self):
         x = ClientConnection("ncalrpc:localhost[DEFAULT]",
@@ -40,14 +40,14 @@ class BareTestCase(samba.tests.TestCase):
                 ("60a15ec5-4de8-11d7-a637-005056a20182", 1),
                 basis_connection=x, lp_ctx=samba.tests.env_loadparm())
         self.assertEquals(24, len(x.request(0, chr(0) * 8)))
-        self.assertEquals("\x01\x00\x00\x00", y.request(0, chr(0) * 4))
+        self.assertEquals(b"\x01\x00\x00\x00", y.request(0, chr(0) * 4))
 
     def test_bare_tcp(self):
         # Connect to the echo pipe
         x = ClientConnection("ncacn_ip_tcp:%s" % os.environ["SERVER"],
                 ("60a15ec5-4de8-11d7-a637-005056a20182", 1),
                 lp_ctx=samba.tests.env_loadparm())
-        self.assertEquals("\x01\x00\x00\x00", x.request(0, chr(0) * 4))
+        self.assertEquals(b"\x01\x00\x00\x00", x.request(0, chr(0) * 4))
 
     def test_two_contexts_tcp(self):
         x = ClientConnection("ncacn_ip_tcp:%s" % os.environ["SERVER"],
@@ -57,4 +57,4 @@ class BareTestCase(samba.tests.TestCase):
                 ("60a15ec5-4de8-11d7-a637-005056a20182", 1),
                 basis_connection=x, lp_ctx=samba.tests.env_loadparm())
         self.assertEquals(24, len(x.request(0, chr(0) * 8)))
-        self.assertEquals("\x01\x00\x00\x00", y.request(0, chr(0) * 4))
+        self.assertEquals(b"\x01\x00\x00\x00", y.request(0, chr(0) * 4))
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 47ff7f24653..95058b1ea2c 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -579,7 +579,7 @@ planoldpythontestsuite("none", "simple", extra_path=["%s/lib/tdb/python/tests" %
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.sam", py3_compatible=True)
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dsdb", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.dsdb_lock")
-planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare")
+planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare", py3_compatible=True)
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix")
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc")
 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd")
-- 
2.13.6


From 6d34ea110f4cd66d5ef1ca4118448fe241ce0c7f Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 16:38:03 +1200
Subject: [PATCH 17/31] selftest: enable py3 for samba.tests.dcerpc.rpcecho

Prefix `b` for bytes.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/dcerpc/rpcecho.py | 6 +++---
 source4/selftest/tests.py            | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/samba/tests/dcerpc/rpcecho.py b/python/samba/tests/dcerpc/rpcecho.py
index 099f8f619ce..33cb867f09c 100644
--- a/python/samba/tests/dcerpc/rpcecho.py
+++ b/python/samba/tests/dcerpc/rpcecho.py
@@ -53,7 +53,7 @@ class RpcEchoTests(RpcInterfaceTestCase):
         self.assertEquals(8 * [0], y.surrounding)
 
     def test_manual_request(self):
-        self.assertEquals("\x01\x00\x00\x00", self.conn.request(0, chr(0) * 4))
+        self.assertEquals(b"\x01\x00\x00\x00", self.conn.request(0, chr(0) * 4))
 
     def test_server_name(self):
         self.assertEquals(None, self.conn.server_name)
@@ -64,8 +64,8 @@ class NdrEchoTests(TestCase):
     def test_info1_push(self):
         x = echo.info1()
         x.v = 42
-        self.assertEquals("\x2a", ndr_pack(x))
+        self.assertEquals(b"\x2a", ndr_pack(x))
 
     def test_info1_pull(self):
-        x = ndr_unpack(echo.info1, "\x42")
+        x = ndr_unpack(echo.info1, b"\x42")
         self.assertEquals(x.v, 66)
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 95058b1ea2c..076c0e28f48 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -619,7 +619,7 @@ planpythontestsuite("none", "samba.tests.samba_tool.help")
 planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.sites")
 planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.dnscmd")
 
-planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.rpcecho")
+planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.rpcecho", py3_compatible=True)
 
 planoldpythontestsuite("nt4_dc", "samba.tests.netbios", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 planoldpythontestsuite("ad_dc:local", "samba.tests.gpo", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
-- 
2.13.6


From f0dd687e2899d5af3b536050925835b30a68694b Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 16:42:54 +1200
Subject: [PATCH 18/31] selftest: enable py3 for samba.tests.dcerpc.registry

No changes needed.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/selftest/tests.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index 076c0e28f48..d5c421e7c7f 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -626,7 +626,10 @@ planoldpythontestsuite("ad_dc:local", "samba.tests.gpo", extra_args=['-U"$USERNA
 planoldpythontestsuite("ad_dc:local", "samba.tests.dckeytab", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 planoldpythontestsuite("ad_dc:local", "samba.tests.smb", extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
 
-planoldpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.registry", extra_args=['-U"$USERNAME%$PASSWORD"'])
+planoldpythontestsuite(
+    "ad_dc_ntvfs:local", "samba.tests.dcerpc.registry",
+    extra_args=['-U"$USERNAME%$PASSWORD"'], py3_compatible=True)
+
 planoldpythontestsuite("ad_dc_ntvfs", "samba.tests.dcerpc.dnsserver", extra_args=['-U"$USERNAME%$PASSWORD"'])
 planoldpythontestsuite("ad_dc", "samba.tests.dcerpc.dnsserver", extra_args=['-U"$USERNAME%$PASSWORD"'])
 planoldpythontestsuite("chgdcpass", "samba.tests.dcerpc.raw_protocol", extra_args=['-U"$USERNAME%$PASSWORD"'])
-- 
2.13.6


From 9314b1b62130a3f8959b0a7c18bcd4678112a930 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 17:17:48 +1200
Subject: [PATCH 19/31] selftest: enable py3 for samba.tests.lsa_string

No change needed.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/selftest/tests.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index d5c421e7c7f..f7942ea632a 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -715,7 +715,10 @@ planoldpythontestsuite("fl2000dc:local",
                        "samba.tests.encrypted_secrets",
                        extra_args=['-U"$USERNAME%$PASSWORD"'])
 
-planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.lsa_string")
+planpythontestsuite("ad_dc_ntvfs:local",
+                    "samba.tests.lsa_string",
+                    py3_compatible=True)
+
 planoldpythontestsuite("ad_dc_ntvfs",
                        "samba.tests.krb5_credentials",
                        extra_args=['-U"$USERNAME%$PASSWORD"'])
-- 
2.13.6


From cd38a869618f664404f92f764a356c8644177432 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Thu, 5 Apr 2018 12:49:09 +1200
Subject: [PATCH 20/31] selftest: enable py3 for samba.tests.source

In py2, `open` has no `encoding` arg, python guesses file encoding from
locale. This could be wrong.

Use `io.open` to open a file, so we can specify encoding in both py2 and
py3.

Also, open file with `r` instead of `rb` for py3.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/source.py | 3 ++-
 selftest/tests.py            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/samba/tests/source.py b/python/samba/tests/source.py
index 3b18a91b5a2..0e0bd5c37cf 100644
--- a/python/samba/tests/source.py
+++ b/python/samba/tests/source.py
@@ -19,6 +19,7 @@
 
 """Source level Python tests."""
 
+import io
 import errno
 import os
 import re
@@ -60,7 +61,7 @@ def get_source_file_contents():
     """Iterate over the contents of all python files."""
     for fname in get_python_source_files():
         try:
-            f = open(fname, 'rb')
+            f = io.open(fname, mode='r', encoding='utf-8')
         except IOError as e:
             if e.errno == errno.ENOENT:
                 warnings.warn("source file %s broken link?" % fname)
diff --git a/selftest/tests.py b/selftest/tests.py
index d866ca9f970..548e68b5c1b 100644
--- a/selftest/tests.py
+++ b/selftest/tests.py
@@ -41,7 +41,7 @@ have_man_pages_support = ("XSLTPROC_MANPAGES" in config_hash)
 with_pam = ("WITH_PAM" in config_hash)
 pam_wrapper_so_path=config_hash["LIBPAM_WRAPPER_SO_PATH"]
 
-planpythontestsuite("none", "samba.tests.source")
+planpythontestsuite("none", "samba.tests.source", py3_compatible=True)
 if have_man_pages_support:
     planpythontestsuite("none", "samba.tests.docs")
 
-- 
2.13.6


From 65f0fa86710d373bcbd342ffba9b6271e6bd7305 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Thu, 5 Apr 2018 13:48:36 +1200
Subject: [PATCH 21/31] selftest: enable py3 for samba.tests.docs

Popen methods will return bytes.
Decode output to string before using.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/docs.py | 23 +++++++++++++----------
 selftest/tests.py          |  2 +-
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/python/samba/tests/docs.py b/python/samba/tests/docs.py
index 3b06be8ad83..39c8f8976cc 100644
--- a/python/samba/tests/docs.py
+++ b/python/samba/tests/docs.py
@@ -196,10 +196,11 @@ class SmbDotConfTests(TestCase):
             p = subprocess.Popen(program + ["-s", self.smbconf,
                     "--section-name", section, "--parameter-name", param],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.topdir).communicate()
-            if p[0].upper().strip() != default.upper():
-                if not (p[0].upper().strip() == "" and default == '""'):
+            result = p[0].decode().upper().strip()
+            if result != default.upper():
+                if not (result == "" and default == '""'):
                     doc_triple = "%s\n      Expected: %s" % (param, default)
-                    failset.add("%s\n      Got: %s" % (doc_triple, p[0].upper().strip()))
+                    failset.add("%s\n      Got: %s" % (doc_triple, result))
 
         if len(failset) > 0:
             self.fail(self._format_message(failset,
@@ -226,10 +227,11 @@ class SmbDotConfTests(TestCase):
                     "--section-name", section, "--parameter-name", param,
                     "--option", "%s = %s" % (param, default)],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.topdir).communicate()
-            if p[0].upper().strip() != default.upper():
-                if not (p[0].upper().strip() == "" and default == '""'):
+            result = p[0].decode().upper().strip()
+            if result != default.upper():
+                if not (result == "" and default == '""'):
                     doc_triple = "%s\n      Expected: %s" % (param, default)
-                    failset.add("%s\n      Got: %s" % (doc_triple, p[0].upper().strip()))
+                    failset.add("%s\n      Got: %s" % (doc_triple, result))
 
         if len(failset) > 0:
             self.fail(self._format_message(failset,
@@ -284,10 +286,11 @@ class SmbDotConfTests(TestCase):
                     "--section-name", section, "--parameter-name", param,
                     "--option", "%s = %s" % (param, value_to_use)],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.topdir).communicate()
-            if p[0].upper().strip() != value_to_use.upper():
+            result = p[0].decode().upper().strip()
+            if result != value_to_use.upper():
                 # currently no way to distinguish command lists
                 if param_type == 'list':
-                    if ", ".join(p[0].upper().strip().split()) == value_to_use.upper():
+                    if ", ".join(result.split()) == value_to_use.upper():
                         continue
 
                 # currently no way to identify octal
@@ -319,7 +322,7 @@ class SmbDotConfTests(TestCase):
 
             # testparm doesn't display a value if they are equivalent
             if (value_to_use.lower() != opposite_value.lower()):
-                for line in p[0].splitlines():
+                for line in p[0].decode().splitlines():
                     if not line.strip().startswith(param):
                         continue
 
@@ -351,7 +354,7 @@ class SmbDotConfTests(TestCase):
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.topdir).communicate()
         output = ""
 
-        for line in p[0].splitlines():
+        for line in p[0].decode().splitlines():
             if line.strip().startswith('#'):
                 continue
             if line.strip().startswith("idmap config *"):
diff --git a/selftest/tests.py b/selftest/tests.py
index 548e68b5c1b..c8f2411f4b1 100644
--- a/selftest/tests.py
+++ b/selftest/tests.py
@@ -43,7 +43,7 @@ pam_wrapper_so_path=config_hash["LIBPAM_WRAPPER_SO_PATH"]
 
 planpythontestsuite("none", "samba.tests.source", py3_compatible=True)
 if have_man_pages_support:
-    planpythontestsuite("none", "samba.tests.docs")
+    planpythontestsuite("none", "samba.tests.docs", py3_compatible=True)
 
 try:
     import testscenarios
-- 
2.13.6


From 3acd0d53f20e0e267b36692296934cf847f67cdf Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Wed, 11 Apr 2018 17:44:26 +1200
Subject: [PATCH 22/31] selftest: enable py3 for samba.tests.krb5_credentials

Fix encoding issue.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/krb5_credentials.py | 7 +++----
 source4/selftest/tests.py              | 2 ++
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/python/samba/tests/krb5_credentials.py b/python/samba/tests/krb5_credentials.py
index cad19da249d..01694fe7009 100644
--- a/python/samba/tests/krb5_credentials.py
+++ b/python/samba/tests/krb5_credentials.py
@@ -93,10 +93,9 @@ class PyKrb5CredentialsTests(TestCase):
         # remove the account if it exists, this will happen if a previous test
         # run failed
         delete_force(self.ldb, self.machine_dn)
-
-        utf16pw = unicode(
-            '"' + self.machine_pass.encode('utf-8') + '"', 'utf-8'
-        ).encode('utf-16-le')
+        # get unicode str for both py2 and py3
+        pass_unicode = self.machine_pass.encode('utf-8').decode('utf-8')
+        utf16pw = u'"{}"'.format(pass_unicode).encode('utf-16-le')
         self.ldb.add({
             "dn": self.machine_dn,
             "objectclass": "computer",
diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py
index f7942ea632a..d1e5bc6a509 100755
--- a/source4/selftest/tests.py
+++ b/source4/selftest/tests.py
@@ -721,7 +721,9 @@ planpythontestsuite("ad_dc_ntvfs:local",
 
 planoldpythontestsuite("ad_dc_ntvfs",
                        "samba.tests.krb5_credentials",
+                       py3_compatible=True,
                        extra_args=['-U"$USERNAME%$PASSWORD"'])
+
 for env in ["ad_dc_ntvfs", "vampire_dc", "promoted_dc"]:
     planoldpythontestsuite(env,
                            "samba.tests.py_credentials",
-- 
2.13.6


From 1afb368a4be63f12d33b3cb51023af7811351ac8 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:48:35 +1200
Subject: [PATCH 23/31] graph: fix divide for py3

`/` will return float other than int in py3.
Use `//` to keep consistent with py2.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/graph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/samba/graph.py b/python/samba/graph.py
index 3b8f61194f4..6cdd2ef77ed 100644
--- a/python/samba/graph.py
+++ b/python/samba/graph.py
@@ -497,7 +497,7 @@ def get_transitive_colourer(colours, n_vertices):
         n = 1 + int(n_vertices ** 0.5)
 
         def f(link):
-            return scale[min(link * m / n, m - 1)]
+            return scale[min(link * m // n, m - 1)]
 
     else:
         def f(link):
-- 
2.13.6


From 7feb7ede4478fc2be47d6671c03279df3d77f64f Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:51:34 +1200
Subject: [PATCH 24/31] graph: fix sort for py3

`sorted` can not sort `None` with str in py3, use the `key` arg to fix.
Sort None as ''.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/graph.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/python/samba/tests/graph.py b/python/samba/tests/graph.py
index 0a95afb1859..2193d422240 100644
--- a/python/samba/tests/graph.py
+++ b/python/samba/tests/graph.py
@@ -101,6 +101,15 @@ class DotFileTests(samba.tests.TestCaseInTempDir):
 
 
 class DistanceTests(samba.tests.TestCase):
+
+    def setUp(self):
+        super(DistanceTests, self).setUp()
+        # a sorted list of colour set names.
+        self.sorted_colour_sets = sorted(
+            graph.COLOUR_SETS,
+            # return '' for None, so it's sortable.
+            key=lambda name: name or '')
+
     def test_simple_distance(self):
         edges = [('ant', 'bat'),
                  ('cat', 'dog'),
@@ -115,7 +124,7 @@ class DistanceTests(samba.tests.TestCase):
                  ('cat', 'dog')]
 
         for utf8 in (True, False):
-            for colour in sorted(graph.COLOUR_SETS):
+            for colour in self.sorted_colour_sets:
                 print('utf8 %s, colour %s' % (utf8, colour))
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
@@ -129,7 +138,7 @@ class DistanceTests(samba.tests.TestCase):
                  ('ant', 'cat')]
 
         for utf8 in (True, False):
-            for colour in sorted(graph.COLOUR_SETS):
+            for colour in self.sorted_colour_sets:
                 print('utf8 %s, colour %s' % (utf8, colour))
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
@@ -144,7 +153,7 @@ class DistanceTests(samba.tests.TestCase):
                  ('dog', 'eel')]
 
         for utf8 in (True, False):
-            for colour in sorted(graph.COLOUR_SETS):
+            for colour in self.sorted_colour_sets:
                 print('utf8 %s, colour %s' % (utf8, colour))
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
-- 
2.13.6


From a8230f671fefe5ab6eba2c2ddff4b9a6148f6074 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:52:47 +1200
Subject: [PATCH 25/31] selftest/graph: enable py3 for samba.tests.graph

Changes are made separatedly in previous commits.
No change needed here.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 selftest/tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selftest/tests.py b/selftest/tests.py
index c8f2411f4b1..185ad37fd4a 100644
--- a/selftest/tests.py
+++ b/selftest/tests.py
@@ -152,7 +152,7 @@ planpythontestsuite("none", "samba.tests.policy", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.kcc.graph", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.kcc.graph_utils", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.kcc.ldif_import_export")
-planpythontestsuite("none", "samba.tests.graph")
+planpythontestsuite("none", "samba.tests.graph", py3_compatible=True)
 plantestsuite("wafsamba.duplicate_symbols", "none", [os.path.join(srcdir(), "buildtools/wafsamba/test_duplicate_symbol.sh")])
 planpythontestsuite("none", "samba.tests.glue", py3_compatible=True)
 planpythontestsuite("none", "samba.tests.tdb_util", py3_compatible=True)
-- 
2.13.6


From 2d165b09f4b178567bf5ce8b49fc18faf3c25e9e Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 12:45:34 +1200
Subject: [PATCH 26/31] kcc/graph_utils: port string.translate for py3

In py3, `str.translate` removed the second positional argument
`deletechars`, which means you can not use it to delete chars from str.
Use `replace` for this case.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/kcc/graph_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/samba/kcc/graph_utils.py b/python/samba/kcc/graph_utils.py
index c564bee7a0a..6e32100e004 100644
--- a/python/samba/kcc/graph_utils.py
+++ b/python/samba/kcc/graph_utils.py
@@ -32,7 +32,7 @@ def write_dot_file(basename, edge_list, vertices=None, label=None,
     s = dot_graph(vertices, edge_list, title=label, **kwargs)
     if label:
         # sanitise DN and guid labels
-        basename += '_' + label.translate(None, ', ')
+        basename += '_' + label.replace(', ', '')
 
     filename = os.path.join(dot_file_dir, "%s.dot" % basename)
     if debug is not None:
-- 
2.13.6


From a1f860480581a79d1592a0b8dc2b4e1499d4ac73 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:42:42 +1200
Subject: [PATCH 27/31] kcc/kcc_utils: convert dict.keys to list

In py3, `dict.keys()` will return a iterator not a list.
Convert it to list to support both py2 and py3.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/kcc/graph_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/samba/kcc/graph_utils.py b/python/samba/kcc/graph_utils.py
index 6e32100e004..086b6512201 100644
--- a/python/samba/kcc/graph_utils.py
+++ b/python/samba/kcc/graph_utils.py
@@ -235,7 +235,7 @@ def verify_graph_directed_double_ring(edges, vertices, edge_vertices):
             raise GraphError("wanted double directed ring, found a leaf node"
                              "(%s)" % vertex)
 
-    for vertex in edge_map.keys():
+    for vertex in list(edge_map.keys()):
         nset = edge_map[vertex]
         if not nset:
             continue
-- 
2.13.6


From 9c2ea811a9ef110b489dba021846dee1638d9cf9 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 12:58:52 +1200
Subject: [PATCH 28/31] kcc/kcc_utils: fix divide for py3

`/` will return float other than int in py3.
Use `//` to keep consistent with py2.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/kcc/kcc_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/samba/kcc/kcc_utils.py b/python/samba/kcc/kcc_utils.py
index a4870a3e7c8..6e15c0b1e31 100644
--- a/python/samba/kcc/kcc_utils.py
+++ b/python/samba/kcc/kcc_utils.py
@@ -1669,7 +1669,7 @@ class Site(object):
         #
         # Note: We don't want to divide by zero here so they must
         #       have meant "f" instead of "o!interSiteTopologyFailover"
-        k_idx = (i_idx + ((self.nt_now - t_time) / f)) % len(D_sort)
+        k_idx = (i_idx + ((self.nt_now - t_time) // f)) % len(D_sort)
 
         # The local writable DC acts as an ISTG for its site if and
         # only if dk is the nTDSDSA object for the local DC. If the
-- 
2.13.6


From 318bf6c1ea9509d161440a79169720d965f4ca35 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 14:51:37 +1200
Subject: [PATCH 29/31] kcc: fix sort for py3

py2:

    list.sort(cmp=None, key=None, reverse=False)
    sorted(iterable[, cmp[, key[, reverse]]])

py3:

    list.sort(key=None, reverse=False)
    sorted(iterable, *, key=None, reverse=False)

The `cmp` arg was removed in py3, make use of `key` arg to work around.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/kcc/__init__.py  | 14 +-------------
 python/samba/kcc/kcc_utils.py |  9 +++------
 2 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/python/samba/kcc/__init__.py b/python/samba/kcc/__init__.py
index f0046630fcf..1a962bb03c1 100644
--- a/python/samba/kcc/__init__.py
+++ b/python/samba/kcc/__init__.py
@@ -46,18 +46,6 @@ from samba.kcc.debug import DEBUG, DEBUG_FN, logger
 from samba.kcc import debug
 
 
-def sort_replica_by_dsa_guid(rep1, rep2):
-    """Helper to sort NCReplicas by their DSA guids
-
-    The guids need to be sorted in their NDR form.
-
-    :param rep1: An NC replica
-    :param rep2: Another replica
-    :return: -1, 0, or 1, indicating sort order.
-    """
-    return cmp(ndr_pack(rep1.rep_dsa_guid), ndr_pack(rep2.rep_dsa_guid))
-
-
 def sort_dsa_by_gc_and_guid(dsa1, dsa2):
     """Helper to sort DSAs by guid global catalog status
 
@@ -2194,7 +2182,7 @@ class KCC(object):
         # on the local DC
         r_list.append(l_of_x)
 
-        r_list.sort(sort_replica_by_dsa_guid)
+        r_list.sort(key=lambda rep: ndr_pack(rep.rep_dsa_guid))
         r_len = len(r_list)
 
         max_node_edges = self.intrasite_max_node_edges(r_len)
diff --git a/python/samba/kcc/kcc_utils.py b/python/samba/kcc/kcc_utils.py
index 6e15c0b1e31..2118570bbfc 100644
--- a/python/samba/kcc/kcc_utils.py
+++ b/python/samba/kcc/kcc_utils.py
@@ -1581,7 +1581,9 @@ class Site(object):
         # Which is a fancy way of saying "sort all the nTDSDSA objects
         # in the site by guid in ascending order".   Place sorted list
         # in D_sort[]
-        D_sort = sorted(self.rw_dsa_table.values(), cmp=sort_dsa_by_guid)
+        D_sort = sorted(
+            self.rw_dsa_table.values(),
+            key=lambda dsa: ndr_pack(dsa.dsa_guid))
 
         # double word number of 100 nanosecond intervals since 1600s
 
@@ -2221,11 +2223,6 @@ def get_dsa_config_rep(dsa):
                    dsa.dsa_dnstr)
 
 
-def sort_dsa_by_guid(dsa1, dsa2):
-    "use ndr_pack for GUID comparison, as appears correct in some places"""
-    return cmp(ndr_pack(dsa1.dsa_guid), ndr_pack(dsa2.dsa_guid))
-
-
 def new_connection_schedule():
     """Create a default schedule for an NTDSConnection or Sitelink. This
     is packed differently from the repltimes schedule used elsewhere
-- 
2.13.6


From a61acfe4a605a1be0dc906542db97f4b83b1e532 Mon Sep 17 00:00:00 2001
From: Joe Guo <joeg at catalyst.net.nz>
Date: Tue, 10 Apr 2018 15:06:51 +1200
Subject: [PATCH 30/31] kcc/graph: add __hash__ to InternalEdge for py3

In py3, if a class defines `__eq__()` but not `__hash__()`, its instances will
not be usable as items in hashable collections, e.g.: set.

Add `__hash__()` to InternalEdge, so it can be added to a set in py3.

Signed-off-by: Joe Guo <joeg at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/kcc/graph.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/python/samba/kcc/graph.py b/python/samba/kcc/graph.py
index a0bff6997db..fb3ca0c81c2 100644
--- a/python/samba/kcc/graph.py
+++ b/python/samba/kcc/graph.py
@@ -805,6 +805,11 @@ class InternalEdge(object):
         self.e_type = eType
         self.site_link = site_link
 
+    def __hash__(self):
+        return hash((
+            self.v1, self.v2, self.red_red, self.repl_info, self.e_type,
+            self.site_link))
+
     def __eq__(self, other):
         return not self < other and not other < self
 
-- 
2.13.6


From 7bc3089d2571496ce5d729cb5d90462b1e9310fc Mon Sep 17 00:00:00 2001
From: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
Date: Thu, 12 Apr 2018 15:57:09 +1200
Subject: [PATCH 31/31] python/tests/graph: actually test graphs, don't print

Signed-off-by: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/graph.py | 397 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 385 insertions(+), 12 deletions(-)

diff --git a/python/samba/tests/graph.py b/python/samba/tests/graph.py
index 2193d422240..7ea53caa734 100644
--- a/python/samba/tests/graph.py
+++ b/python/samba/tests/graph.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Test graph dot file generation
 #
 # Copyright (C) Andrew Bartlett 2018.
@@ -93,7 +94,6 @@ class DotFileTests(samba.tests.TestCaseInTempDir):
                 g = graph.dot_graph(vertices, edges,
                                     directed=directed,
                                     title=title)
-                print(g)
                 lines = g.split('\n')
                 self.assertHeader(lines, title, directed)
                 self.assertVertices(lines[7:], vertices)
@@ -123,27 +123,269 @@ class DistanceTests(samba.tests.TestCase):
                  ('cat', 'ant'),
                  ('cat', 'dog')]
 
+        expected = {
+                "utf8 True, colour None": '''
+                 destination
+         ╭────── ant
+         │╭───── bat
+         ││╭──── cat
+         │││╭─── dog
+         ││││╭── elephant
+  source │││││╭─ frog
+     ant ·1221-
+     bat 3·211-
+     cat 12·12-
+     dog ---·--
+elephant 2311·-
+    frog 23121·''',
+                'utf8 True, colour ansi': '''
+                 destination
+         ╭────── ant
+         │╭───── bat
+         ││╭──── cat
+         │││╭─── dog
+         ││││╭── elephant
+  source │││││╭─ frog
+     ant ·1221-
+     bat 3·211-
+     cat 12·12-
+     dog ---·--
+elephant 2311·-
+    frog 23121·
+                ''',
+                'utf8 True, colour ansi-heatmap': '''
+                 destination
+         ╭────── ant
+         │╭───── bat
+         ││╭──── cat
+         │││╭─── dog
+         ││││╭── elephant
+  source │││││╭─ frog
+     ant ·1221-
+     bat 3·211-
+     cat 12·12-
+     dog ---·--
+elephant 2311·-
+    frog 23121·
+                ''',
+                'utf8 True, colour xterm-256color': '''
+                 destination
+         ╭────── ant
+         │╭───── bat
+         ││╭──── cat
+         │││╭─── dog
+         ││││╭── elephant
+  source │││││╭─ frog
+     ant ·1221-
+     bat 3·211-
+     cat 12·12-
+     dog ---·--
+elephant 2311·-
+    frog 23121·
+                ''',
+'utf8 True, colour xterm-256color-heatmap': '''
+                 destination
+         ╭────── ant
+         │╭───── bat
+         ││╭──── cat
+         │││╭─── dog
+         ││││╭── elephant
+  source │││││╭─ frog
+     ant ·1221-
+     bat 3·211-
+     cat 12·12-
+     dog ---·--
+elephant 2311·-
+    frog 23121·
+''',
+'utf8 False, colour None': '''
+                 destination
+         ,------ ant
+         |,----- bat
+         ||,---- cat
+         |||,--- dog
+         ||||,-- elephant
+  source |||||,- frog
+     ant 01221-
+     bat 30211-
+     cat 12012-
+     dog ---0--
+elephant 23110-
+    frog 231210
+''',
+'utf8 False, colour ansi': '''
+                 destination
+         ,------ ant
+         |,----- bat
+         ||,---- cat
+         |||,--- dog
+         ||||,-- elephant
+  source |||||,- frog
+     ant 01221-
+     bat 30211-
+     cat 12012-
+     dog ---0--
+elephant 23110-
+    frog 231210
+''',
+'utf8 False, colour ansi-heatmap': '''
+                 destination
+         ,------ ant
+         |,----- bat
+         ||,---- cat
+         |||,--- dog
+         ||||,-- elephant
+  source |||||,- frog
+     ant 01221-
+     bat 30211-
+     cat 12012-
+     dog ---0--
+elephant 23110-
+    frog 231210
+''',
+'utf8 False, colour xterm-256color': '''
+                 destination
+         ,------ ant
+         |,----- bat
+         ||,---- cat
+         |||,--- dog
+         ||||,-- elephant
+  source |||||,- frog
+     ant 01221-
+     bat 30211-
+     cat 12012-
+     dog ---0--
+elephant 23110-
+    frog 231210
+''',
+'utf8 False, colour xterm-256color-heatmap': '''
+                 destination
+         ,------ ant
+         |,----- bat
+         ||,---- cat
+         |||,--- dog
+         ||||,-- elephant
+  source |||||,- frog
+     ant 01221-
+     bat 30211-
+     cat 12012-
+     dog ---0--
+elephant 23110-
+    frog 231210
+'''
+        }
         for utf8 in (True, False):
             for colour in self.sorted_colour_sets:
-                print('utf8 %s, colour %s' % (utf8, colour))
+                k = 'utf8 %s, colour %s' % (utf8, colour)
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
-                print(s)
-                print()
+                self.assertStringsEqual(s, expected[k], strip=True,
+                                        msg='Wrong output: %s\n\n%s' % (k, s))
 
     def test_simple_distance2(self):
         edges = [('ant', 'bat'),
                  ('cat', 'bat'),
                  ('bat', 'ant'),
                  ('ant', 'cat')]
-
+        expected = {
+            'utf8 True, colour None': '''
+            destination
+       ╭─── ant
+       │╭── bat
+source ││╭─ cat
+   ant ·11
+   bat 1·2
+   cat 21·
+            ''',
+            'utf8 True, colour ansi': '''
+            destination
+       ╭─── ant
+       │╭── bat
+source ││╭─ cat
+   ant ·11
+   bat 1·2
+   cat 21·
+            ''',
+'utf8 True, colour ansi-heatmap': '''
+            destination
+       ╭─── ant
+       │╭── bat
+source ││╭─ cat
+   ant ·11
+   bat 1·2
+   cat 21·
+        ''',
+'utf8 True, colour xterm-256color': '''
+            destination
+       ╭─── ant
+       │╭── bat
+source ││╭─ cat
+   ant ·11
+   bat 1·2
+   cat 21·
+''',
+'utf8 True, colour xterm-256color-heatmap': '''
+            destination
+       ╭─── ant
+       │╭── bat
+source ││╭─ cat
+   ant ·11
+   bat 1·2
+   cat 21·
+''',
+'utf8 False, colour None': '''
+            destination
+       ,--- ant
+       |,-- bat
+source ||,- cat
+   ant 011
+   bat 102
+   cat 210
+''',
+'utf8 False, colour ansi': '''
+            destination
+       ,--- ant
+       |,-- bat
+source ||,- cat
+   ant 011
+   bat 102
+   cat 210
+''',
+'utf8 False, colour ansi-heatmap': '''
+            destination
+       ,--- ant
+       |,-- bat
+source ||,- cat
+   ant 011
+   bat 102
+   cat 210
+''',
+'utf8 False, colour xterm-256color': '''
+            destination
+       ,--- ant
+       |,-- bat
+source ||,- cat
+   ant 011
+   bat 102
+   cat 210
+''',
+'utf8 False, colour xterm-256color-heatmap': '''
+            destination
+       ,--- ant
+       |,-- bat
+source ||,- cat
+   ant 011
+   bat 102
+   cat 210
+'''
+        }
         for utf8 in (True, False):
             for colour in self.sorted_colour_sets:
-                print('utf8 %s, colour %s' % (utf8, colour))
+                k = 'utf8 %s, colour %s' % (utf8, colour)
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
-                print(s)
-                print()
+                self.assertStringsEqual(s, expected[k], strip=True,
+                                        msg='Wrong output: %s\n\n%s' % (k, s))
 
     def test_simple_distance3(self):
         edges = [('ant', 'bat'),
@@ -151,11 +393,142 @@ class DistanceTests(samba.tests.TestCase):
                  ('cat', 'dog'),
                  ('dog', 'ant'),
                  ('dog', 'eel')]
-
+        expected = {
+'utf8 True, colour None': '''
+              destination
+       ╭───── ant
+       │╭──── bat
+       ││╭─── cat
+       │││╭── dog
+source ││││╭─ eel
+   ant ·1234
+   bat 3·123
+   cat 23·12
+   dog 123·1
+   eel ----·
+''',
+'utf8 True, colour ansi': '''
+              destination
+       ╭───── ant
+       │╭──── bat
+       ││╭─── cat
+       │││╭── dog
+source ││││╭─ eel
+   ant ·1234
+   bat 3·123
+   cat 23·12
+   dog 123·1
+   eel ----·
+''',
+'utf8 True, colour ansi-heatmap': '''
+              destination
+       ╭───── ant
+       │╭──── bat
+       ││╭─── cat
+       │││╭── dog
+source ││││╭─ eel
+   ant ·1234
+   bat 3·123
+   cat 23·12
+   dog 123·1
+   eel ----·
+''',
+'utf8 True, colour xterm-256color': '''
+              destination
+       ╭───── ant
+       │╭──── bat
+       ││╭─── cat
+       │││╭── dog
+source ││││╭─ eel
+   ant ·1234
+   bat 3·123
+   cat 23·12
+   dog 123·1
+   eel ----·
+''',
+'utf8 True, colour xterm-256color-heatmap': '''
+              destination
+       ╭───── ant
+       │╭──── bat
+       ││╭─── cat
+       │││╭── dog
+source ││││╭─ eel
+   ant ·1234
+   bat 3·123
+   cat 23·12
+   dog 123·1
+   eel ----·
+''',
+'utf8 False, colour None': '''
+              destination
+       ,----- ant
+       |,---- bat
+       ||,--- cat
+       |||,-- dog
+source ||||,- eel
+   ant 01234
+   bat 30123
+   cat 23012
+   dog 12301
+   eel ----0
+''',
+'utf8 False, colour ansi': '''
+              destination
+       ,----- ant
+       |,---- bat
+       ||,--- cat
+       |||,-- dog
+source ||||,- eel
+   ant 01234
+   bat 30123
+   cat 23012
+   dog 12301
+   eel ----0
+''',
+'utf8 False, colour ansi-heatmap': '''
+              destination
+       ,----- ant
+       |,---- bat
+       ||,--- cat
+       |||,-- dog
+source ||||,- eel
+   ant 01234
+   bat 30123
+   cat 23012
+   dog 12301
+   eel ----0
+''',
+'utf8 False, colour xterm-256color':
+'''              destination
+       ,----- ant
+       |,---- bat
+       ||,--- cat
+       |||,-- dog
+source ||||,- eel
+   ant 01234
+   bat 30123
+   cat 23012
+   dog 12301
+   eel ----0
+''',
+'utf8 False, colour xterm-256color-heatmap': '''
+              destination
+       ,----- ant
+       |,---- bat
+       ||,--- cat
+       |||,-- dog
+source ||||,- eel
+   ant 01234
+   bat 30123
+   cat 23012
+   dog 12301
+   eel ----0
+'''
+        }
         for utf8 in (True, False):
             for colour in self.sorted_colour_sets:
-                print('utf8 %s, colour %s' % (utf8, colour))
+                k = 'utf8 %s, colour %s' % (utf8, colour)
                 s = graph.distance_matrix(None, edges, utf8=utf8,
                                           colour=colour)
-                print(s)
-                print()
+                self.assertStringsEqual(s, expected[k], strip=True,
+                                        msg='Wrong output: %s\n\n%s' % (k, s))
-- 
2.13.6



More information about the samba-technical mailing list