From 5e5858448ca5dc0aa597539f891187c8ea9a287f Mon Sep 17 00:00:00 2001 From: Douglas Bagnall Date: Fri, 20 Apr 2018 10:52:31 +1200 Subject: [PATCH 01/62] samba-tool visualize: use correct DC in graph label We have been using a random DC (depending to hash order, which was not random enough on Python 2.7 to affect the tests). Reported-by: Noel Power Signed-off-by: Douglas Bagnall --- python/samba/netcmd/visualize.py | 6 +++--- python/samba/tests/samba_tool/visualize.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index b983e1891a9..6f880ae61e8 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -392,7 +392,7 @@ def run(self, H=None, output=None, shorten_names=False, lp = sambaopts.get_loadparm() creds = credopts.get_credentials(lp, fallback_machine=True) local_kcc, dsas = self.get_kcc_and_dsas(H, lp, creds) - + local_dsa_dn = local_kcc.my_dsa_dnstr.split(',', 1)[1] vertices = set() attested_edges = [] for dsa_dn in dsas: @@ -452,7 +452,7 @@ def run(self, H=None, output=None, shorten_names=False, # If we are not talking to remote servers, we list all # the connections. graph_edges = edges.keys() - title = 'NTDS Connections known to %s' % dsa_dn + title = 'NTDS Connections known to %s' % local_dsa_dn epilog = '' else: @@ -556,7 +556,7 @@ def run(self, H=None, output=None, shorten_names=False, if talk_to_remote: title = 'NTDS Connections' else: - title = 'NTDS Connections known to %s' % dsa_dn + title = 'NTDS Connections known to %s' % local_dsa_dn s = dot_graph(sorted(vertices), dot_edges, directed=True, diff --git a/python/samba/tests/samba_tool/visualize.py b/python/samba/tests/samba_tool/visualize.py index c00c6ea63b0..a90c77364b7 100644 --- a/python/samba/tests/samba_tool/visualize.py +++ b/python/samba/tests/samba_tool/visualize.py @@ -205,12 +205,14 @@ def test_utf8(self): self.assertTrue(c not in ascii, 'ASCII should not contain %s' % c) def test_forced_local_dsa(self): - # the forced_local_dsa shouldn't make any difference + # the forced_local_dsa shouldn't make any difference, except + # for the title line. result, target, err = self.runsubcmd("visualize", "ntdsconn", '-H', self.dburl, '--color=no', '-S') self.assertCmdSuccess(result, target, err) files = [] + target = target.strip().split('\n', 1)[1] for cn, site in MULTISITE_LDIF_DSAS: dsa = DN_TEMPLATE % (cn, site) samdb, dbfile = samdb_from_ldif(MULTISITE_LDIF, @@ -222,7 +224,10 @@ def test_forced_local_dsa(self): '-H', 'tdb://' + dbfile, '--color=no', '-S') self.assertCmdSuccess(result, out, err) - self.assertStringsEqual(target, out) + # Separate out the title line, which will differ in the DN. + title, body = out.strip().split('\n', 1) + self.assertStringsEqual(target, body) + self.assertIn(cn, title) files.append(dbfile) self.remove_files(*files) @@ -342,7 +347,7 @@ def test_dot_ntdsconn_disconnected_to_file(self): EXPECTED_DOT_MULTISITE_NO_KEY = r"""/* generated by samba */ digraph A_samba_tool_production { -label="NTDS Connections known to CN=WIN07,CN=Servers,CN=Site-4,CN=Sites,CN=Configuration,DC=ad,DC=samba,DC=example,DC=com"; +label="NTDS Connections known to CN=WIN01,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=ad,DC=samba,DC=example,DC=com"; fontsize=10; node[fontname=Helvetica; fontsize=10]; @@ -382,7 +387,7 @@ def test_dot_ntdsconn_disconnected_to_file(self): EXPECTED_DOT_NTDSCONN_DISCONNECTED = r"""/* generated by samba */ digraph A_samba_tool_production { -label="NTDS Connections known to CN=SERVER2,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com"; +label="NTDS Connections known to CN=LOCALDC,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com"; fontsize=10; node[fontname=Helvetica; fontsize=10]; @@ -435,7 +440,7 @@ def test_dot_ntdsconn_disconnected_to_file(self): """ EXPECTED_DISTANCE_GRAPH_WITH_KEY = """ -NTDS Connections known to CN=SERVER2,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com +NTDS Connections known to CN=LOCALDC,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com destination ,-------- *,CN=CLIENT+ |,------- *,CN=LOCALDC+ From a9e8fed668082b2c969d8dbe05921488a1c08384 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 11:19:10 +0100 Subject: [PATCH 02/62] python/samba: Add some compatability PY2/PY3 functions I hope these changes are a short term interim solution for the absence of the 'six' module/library. I also hope that soon this module can be removed and be replaced by usage of six. Signed-off-by: Noel Power --- python/samba/compat.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/samba/compat.py b/python/samba/compat.py index 667a1a443b1..d17dc3c2ce9 100644 --- a/python/samba/compat.py +++ b/python/samba/compat.py @@ -22,8 +22,20 @@ PY3 = sys.version_info[0] == 3 if PY3: + # compat functions + from urllib.parse import quote as urllib_quote + from urllib.request import urlopen as urllib_urlopen + + # compat types integer_types = int, + string_types = str text_type = str else: + # compat functions + from urllib import quote as urllib_quote + from urllib import urlopen as urllib_urlopen + + # compat types integer_types = (int, long) + string_types = basestring text_type = unicode From 12b3306790c0880d76c85993cfd8821deba4cf34 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 12 Apr 2018 14:46:59 +0100 Subject: [PATCH 03/62] lib/ldb: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- lib/ldb/pyldb.c | 10 +++++----- lib/ldb/pyldb_util.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/ldb/pyldb.c b/lib/ldb/pyldb.c index 67ecafbc420..110ec8e60ad 100644 --- a/lib/ldb/pyldb.c +++ b/lib/ldb/pyldb.c @@ -1078,7 +1078,7 @@ static const char **PyList_AsStrList(TALLOC_CTX *mem_ctx, PyObject *list, const char *str = NULL; Py_ssize_t size; PyObject *item = PyList_GetItem(list, i); - if (!PyStr_Check(item)) { + if (!(PyStr_Check(item) || PyUnicode_Check(item))) { PyErr_Format(PyExc_TypeError, "%s should be strings", paramname); talloc_free(ret); return NULL; @@ -2861,7 +2861,7 @@ static struct ldb_message_element *PyObject_AsMessageElement( me->name = talloc_strdup(me, attr_name); me->flags = flags; - if (PyBytes_Check(set_obj) || PyStr_Check(set_obj)) { + if (PyBytes_Check(set_obj) || PyUnicode_Check(set_obj)) { me->num_values = 1; me->values = talloc_array(me, struct ldb_val, me->num_values); if (PyBytes_Check(set_obj)) { @@ -2897,7 +2897,7 @@ static struct ldb_message_element *PyObject_AsMessageElement( return NULL; } msg = _msg; - } else if (PyStr_Check(obj)) { + } else if (PyUnicode_Check(obj)) { msg = PyStr_AsUTF8AndSize(obj, &size); if (msg == NULL) { talloc_free(me); @@ -3069,7 +3069,7 @@ static PyObject *py_ldb_msg_element_new(PyTypeObject *type, PyObject *args, PyOb if (py_elements != NULL) { Py_ssize_t i; - if (PyBytes_Check(py_elements) || PyStr_Check(py_elements)) { + if (PyBytes_Check(py_elements) || PyUnicode_Check(py_elements)) { char *_msg = NULL; el->num_values = 1; el->values = talloc_array(el, struct ldb_val, 1); @@ -3110,7 +3110,7 @@ static PyObject *py_ldb_msg_element_new(PyTypeObject *type, PyObject *args, PyOb char *_msg = NULL; result = PyBytes_AsStringAndSize(item, &_msg, &size); msg = _msg; - } else if (PyStr_Check(item)) { + } else if (PyUnicode_Check(item)) { msg = PyStr_AsUTF8AndSize(item, &size); result = (msg == NULL) ? -1 : 0; } else { diff --git a/lib/ldb/pyldb_util.c b/lib/ldb/pyldb_util.c index 3bda1dbc2a8..46ee4034122 100644 --- a/lib/ldb/pyldb_util.c +++ b/lib/ldb/pyldb_util.c @@ -70,7 +70,7 @@ bool pyldb_Object_AsDn(TALLOC_CTX *mem_ctx, PyObject *object, struct ldb_dn *odn; PyTypeObject *PyLdb_Dn_Type; - if (ldb_ctx != NULL && PyStr_Check(object)) { + if (ldb_ctx != NULL && (PyStr_Check(object) || PyUnicode_Check(object))) { odn = ldb_dn_new(mem_ctx, ldb_ctx, PyStr_AsUTF8(object)); *dn = odn; return true; From d253bbd5114e667ef4925fbced0620a538e6f752 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:17:20 +0100 Subject: [PATCH 04/62] lib/tevent: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- lib/tevent/pytevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/pytevent.c b/lib/tevent/pytevent.c index 10d8a22a8cf..369ec6e02c8 100644 --- a/lib/tevent/pytevent.c +++ b/lib/tevent/pytevent.c @@ -188,7 +188,7 @@ static PyObject *py_register_backend(PyObject *self, PyObject *args) return NULL; } - if (!PyStr_Check(name)) { + if (!(PyStr_Check(name) || PyUnicode_Check(name))) { PyErr_SetNone(PyExc_TypeError); Py_DECREF(name); return NULL; From 147f3bd1d78a7d3453fd0ec138066f83aa7b0e97 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:32:15 +0100 Subject: [PATCH 05/62] libcli/nbt: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- libcli/nbt/pynbt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libcli/nbt/pynbt.c b/libcli/nbt/pynbt.c index 254c98a4b89..032561a4bd8 100644 --- a/libcli/nbt/pynbt.c +++ b/libcli/nbt/pynbt.c @@ -57,7 +57,7 @@ static PyObject *py_nbt_node_init(PyTypeObject *self, PyObject *args, PyObject * static bool PyObject_AsDestinationTuple(PyObject *obj, const char **dest_addr, uint16_t *dest_port) { - if (PyStr_Check(obj)) { + if (PyStr_Check(obj) || PyUnicode_Check(obj)) { *dest_addr = PyStr_AsString(obj); *dest_port = NBT_NAME_SERVICE_PORT; return true; @@ -69,7 +69,7 @@ static bool PyObject_AsDestinationTuple(PyObject *obj, const char **dest_addr, u return false; } - if (!PyStr_Check(PyTuple_GetItem(obj, 0))) { + if (!(PyStr_Check(PyTuple_GetItem(obj, 0)) || PyUnicode_Check(PyTuple_GetItem(obj, 0)))) { PyErr_SetString(PyExc_TypeError, "Destination tuple first element not string"); return false; } @@ -111,7 +111,7 @@ static bool PyObject_AsNBTName(PyObject *obj, struct nbt_name_socket *name_socke } } - if (PyStr_Check(obj)) { + if (PyStr_Check(obj) || PyUnicode_Check(obj)) { /* FIXME: Parse string to be able to interpret things like RHONWYN<02> ? */ name->name = PyStr_AsString(obj); name->scope = NULL; From 32f5bddd64d878dd8267f9e6bcff80995a6019ae Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:33:10 +0100 Subject: [PATCH 06/62] s4/auth: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- source4/auth/pyauth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/auth/pyauth.c b/source4/auth/pyauth.c index 4cb12f882bc..fc22637564d 100644 --- a/source4/auth/pyauth.c +++ b/source4/auth/pyauth.c @@ -184,7 +184,7 @@ static const char **PyList_AsStringList(TALLOC_CTX *mem_ctx, PyObject *list, const char *value; Py_ssize_t size; PyObject *item = PyList_GetItem(list, i); - if (!PyStr_Check(item)) { + if (!(PyStr_Check(item) || PyUnicode_Check(item))) { PyErr_Format(PyExc_TypeError, "%s should be strings", paramname); return NULL; } From 52c88815c6fb5a167fb7aba35a141bdcc2581b63 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:33:47 +0100 Subject: [PATCH 07/62] s4/dsdb: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- source4/dsdb/pydsdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/dsdb/pydsdb.c b/source4/dsdb/pydsdb.c index 8d84a16dd18..d9177bbd1d7 100644 --- a/source4/dsdb/pydsdb.c +++ b/source4/dsdb/pydsdb.c @@ -580,7 +580,7 @@ static PyObject *py_dsdb_DsReplicaAttribute(PyObject *self, PyObject *args) for (i = 0; i < el->num_values; i++) { PyObject *item = PyList_GetItem(el_list, i); - if (!PyStr_Check(item)) { + if (!(PyStr_Check(item) || PyUnicode_Check(item))) { PyErr_Format(PyExc_TypeError, "ldif_elements should be strings"); talloc_free(tmp_ctx); return NULL; From eb3b86b962bcccb77575b7bece606b0c246697a7 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:34:19 +0100 Subject: [PATCH 08/62] s4/librpc: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- source4/librpc/rpc/pyrpc.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source4/librpc/rpc/pyrpc.c b/source4/librpc/rpc/pyrpc.c index 8b817b8b46d..e86ea0e94aa 100644 --- a/source4/librpc/rpc/pyrpc.c +++ b/source4/librpc/rpc/pyrpc.c @@ -50,28 +50,32 @@ static bool ndr_syntax_from_py_object(PyObject *object, struct ndr_syntax_id *sy { ZERO_STRUCTP(syntax_id); - if (PyStr_Check(object)) { + if (PyStr_Check(object) || PyUnicode_Check(object)) { return PyString_AsGUID(object, &syntax_id->uuid); } else if (PyTuple_Check(object)) { + PyObject *item = NULL; if (PyTuple_Size(object) < 1 || PyTuple_Size(object) > 2) { PyErr_SetString(PyExc_ValueError, "Syntax ID tuple has invalid size"); return false; } - if (!PyStr_Check(PyTuple_GetItem(object, 0))) { + item = PyTuple_GetItem(object, 0); + if (!(PyStr_Check(item) || PyUnicode_Check(item))) { PyErr_SetString(PyExc_ValueError, "Expected GUID as first element in tuple"); return false; } - if (!PyString_AsGUID(PyTuple_GetItem(object, 0), &syntax_id->uuid)) + if (!PyString_AsGUID(item, &syntax_id->uuid)) { return false; + } - if (!PyInt_Check(PyTuple_GetItem(object, 1))) { + item = PyTuple_GetItem(object, 1); + if (!PyInt_Check(item)) { PyErr_SetString(PyExc_ValueError, "Expected version as second element in tuple"); return false; } - syntax_id->if_version = PyInt_AsLong(PyTuple_GetItem(object, 1)); + syntax_id->if_version = PyInt_AsLong(item); return true; } From 98b870031160dbd962968e46ad14e035d93d6f64 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 13 Apr 2018 17:34:40 +0100 Subject: [PATCH 09/62] s4/param: Additionally accept unicode as string param in Py2 With the changes to make samba python code Py2/Py3 compatible there now are many instances where string content is decoded. Decoded string variables in Py2 are returned as the unicode type. Many Py2 c-module functions that take string arguments only check for the string type. However now it's quite possibe the content formally passed as a string argument is now passed as unicode after being decoded, such arguments are rejected and code can fail subtly. This only affects places where the type is directly checked e.g. via PyStr_Check etc. arguments that are parsed by ParseTuple* functions generally already accept both string and unicode (if 's', 'z', 's*' format specifiers are used) Signed-off-by: Noel Power --- source4/param/pyparam.c | 2 +- source4/param/pyparam_util.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/param/pyparam.c b/source4/param/pyparam.c index 18d7017b9c6..e7e908fcac3 100644 --- a/source4/param/pyparam.c +++ b/source4/param/pyparam.c @@ -456,7 +456,7 @@ static Py_ssize_t py_lp_ctx_len(PyObject *self) static PyObject *py_lp_ctx_getitem(PyObject *self, PyObject *name) { struct loadparm_service *service; - if (!PyStr_Check(name)) { + if (!(PyStr_Check(name) || PyUnicode_Check(name))) { PyErr_SetString(PyExc_TypeError, "Only string subscripts are supported"); return NULL; } diff --git a/source4/param/pyparam_util.c b/source4/param/pyparam_util.c index 512a8b1cdb7..917caf809e5 100644 --- a/source4/param/pyparam_util.c +++ b/source4/param/pyparam_util.c @@ -34,7 +34,7 @@ _PUBLIC_ struct loadparm_context *lpcfg_from_py_object(TALLOC_CTX *mem_ctx, PyOb PyTypeObject *lp_type; bool is_lpobj; - if (PyStr_Check(py_obj)) { + if (PyStr_Check(py_obj) || PyUnicode_Check(py_obj)) { lp_ctx = loadparm_init_global(false); if (lp_ctx == NULL) { return NULL; From 59cff30d1ba92625eb257a8ad6d0d823ec2923e3 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 10 Apr 2018 15:28:59 +0100 Subject: [PATCH 10/62] python/samba: Port to py3 code exercised by samba.tests.dsdb_lock Signed-off-by: Noel Power --- python/samba/__init__.py | 7 ++-- python/samba/dbchecker.py | 7 ++-- python/samba/descriptor.py | 2 +- python/samba/idmap.py | 2 +- python/samba/ms_display_specifiers.py | 3 +- python/samba/ms_schema.py | 11 ++++--- python/samba/netcmd/fsmo.py | 2 +- python/samba/provision/__init__.py | 60 ++++++++++++++++++----------------- python/samba/provision/sambadns.py | 12 +++---- python/samba/samdb.py | 4 +-- python/samba/schema.py | 7 ++-- 11 files changed, 62 insertions(+), 55 deletions(-) diff --git a/python/samba/__init__.py b/python/samba/__init__.py index f62f5e39202..54fd638eab1 100644 --- a/python/samba/__init__.py +++ b/python/samba/__init__.py @@ -29,6 +29,8 @@ import samba.param from samba import _glue from samba._ldb import Ldb as _Ldb +from samba.compat import string_types + def source_tree_topdir(): @@ -247,10 +249,9 @@ def substitute_var(text, values): :param text: Text in which to subsitute. :param values: Dictionary with keys and values. """ - for (name, value) in values.items(): - assert isinstance(name, str), "%r is not a string" % name - assert isinstance(value, str), "Value %r for %s is not a string" % (value, name) + assert isinstance(name, string_types), "%r is not a string" % name + assert isinstance(value, string_types), "Value %r for %s is not a string" % (value, name) text = text.replace("${%s}" % name, value) return text diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index fb968a56481..e5906523e93 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -144,6 +144,7 @@ def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, for nc in self.ncs: try: + nc = nc.decode('utf8') dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc), dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER) self.deleted_objects_containers.append(dn) @@ -178,13 +179,13 @@ def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, res = self.samdb.search(base=ldb.Dn(self.samdb, self.samdb.get_serverName()), scope=ldb.SCOPE_BASE, attrs=["serverReference"]) # 2. Get server reference - self.server_ref_dn = ldb.Dn(self.samdb, res[0]['serverReference'][0]) + self.server_ref_dn = ldb.Dn(self.samdb, res[0]['serverReference'][0].decode('utf8')) # 3. Get RID Set res = self.samdb.search(base=self.server_ref_dn, scope=ldb.SCOPE_BASE, attrs=['rIDSetReferences']) if "rIDSetReferences" in res[0]: - self.rid_set_dn = ldb.Dn(self.samdb, res[0]['rIDSetReferences'][0]) + self.rid_set_dn = ldb.Dn(self.samdb, res[0]['rIDSetReferences'][0].decode('utf8')) else: self.rid_set_dn = None @@ -1178,7 +1179,7 @@ def check_dn(self, obj, attrname, syntax_oid): obj[attrname] = ldb.MessageElement(vals, 0, attrname) for val in obj[attrname]: - dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid) + dsdb_dn = dsdb_Dn(self.samdb, val.decode('utf8'), syntax_oid) # all DNs should have a GUID component guid = dsdb_dn.dn.get_extended_component("GUID") diff --git a/python/samba/descriptor.py b/python/samba/descriptor.py index a9c5e1580f0..e4650b294c4 100644 --- a/python/samba/descriptor.py +++ b/python/samba/descriptor.py @@ -413,7 +413,7 @@ def get_wellknown_sds(samdb): attrs=["namingContexts"]) for nc in current[0]["namingContexts"]: - + nc = nc.decode('utf8') dnsforestdn = ldb.Dn(samdb, "DC=ForestDnsZones,%s" % (str(samdb.get_root_basedn()))) if ldb.Dn(samdb, nc) == dnsforestdn: c = (ldb.Dn(samdb, "%s" % str(dnsforestdn)), get_dns_partition_descriptor) diff --git a/python/samba/idmap.py b/python/samba/idmap.py index 0cb729fbc29..a618347fa21 100644 --- a/python/samba/idmap.py +++ b/python/samba/idmap.py @@ -95,4 +95,4 @@ def setup_name_mapping(self, sid, type, unixid=None): cn: %s """ % (sid, unixid, sid, type_string, sid) - self.add(self.parse_ldif(mod).next()[1]) + self.add(next(self.parse_ldif(mod))[1]) diff --git a/python/samba/ms_display_specifiers.py b/python/samba/ms_display_specifiers.py index 0d7b39aaae9..3cba837c652 100644 --- a/python/samba/ms_display_specifiers.py +++ b/python/samba/ms_display_specifiers.py @@ -169,7 +169,8 @@ def read_ms_ldif(filename): out = [] - f = open(filename, "rU") + from io import open + f = open(filename, "r", encoding='latin-1') for entry in __read_raw_entries(f): out.append(__write_ldif_one(__transform_entry(entry))) diff --git a/python/samba/ms_schema.py b/python/samba/ms_schema.py index 889b7f5ef22..3e92393a4bf 100644 --- a/python/samba/ms_schema.py +++ b/python/samba/ms_schema.py @@ -21,6 +21,7 @@ import re import base64 import uuid +from samba.compat import string_types bitFields = {} @@ -172,7 +173,7 @@ def fix_dn(dn): def __convert_bitfield(key, value): """Evaluate the OR expression in 'value'""" - assert(isinstance(value, str)) + assert(isinstance(value, string_types)) value = value.replace("\n ", "") value = value.replace(" ", "") @@ -194,7 +195,7 @@ def __write_ldif_one(entry): out = [] for l in entry: - if isinstance(l[1], str): + if isinstance(l[1], string_types): vl = [l[1]] else: vl = l[1] @@ -246,8 +247,7 @@ def __transform_entry(entry, objectClass): if not l[2]: l[1] = oMObjectClassBER[l[1].strip()] l[2] = True - - if isinstance(l[1], str): + if isinstance(l[1], string_types): l[1] = fix_dn(l[1]) if key == 'dn': @@ -286,8 +286,9 @@ def __parse_schema_file(filename, objectClass): """Load and transform a schema file.""" out = [] + from io import open + f = open(filename, "r", encoding='latin-1') - f = open(filename, "rU") for entry in __read_raw_entries(f): out.append(__write_ldif_one(__transform_entry(entry, objectClass))) diff --git a/python/samba/netcmd/fsmo.py b/python/samba/netcmd/fsmo.py index 41e45592b46..14b8fa6c791 100644 --- a/python/samba/netcmd/fsmo.py +++ b/python/samba/netcmd/fsmo.py @@ -47,7 +47,7 @@ def get_fsmo_roleowner(samdb, roledn, role): raise if 'fSMORoleOwner' in res[0]: - master_owner = (ldb.Dn(samdb, res[0]["fSMORoleOwner"][0])) + master_owner = (ldb.Dn(samdb, res[0]["fSMORoleOwner"][0].decode('utf8'))) else: master_owner = None diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py index 1709e75184e..fc308fcad26 100644 --- a/python/samba/provision/__init__.py +++ b/python/samba/provision/__init__.py @@ -25,7 +25,8 @@ """Functions for setting up a Samba configuration.""" __docformat__ = "restructuredText" - +from samba.compat import string_types +from samba.compat import urllib_quote from base64 import b64encode import errno import os @@ -37,7 +38,6 @@ import time import uuid import socket -import urllib import string import tempfile import samba.dsdb @@ -1070,7 +1070,7 @@ def setup_encrypted_secrets_key(path): finally: os.umask(umask_original) - with os.fdopen(fd, 'w') as f: + with os.fdopen(fd, 'wb') as f: key = samba.generate_random_bytes(16) f.write(key) @@ -1144,7 +1144,7 @@ def setup_self_join(samdb, admin_session_info, names, fill, machinepass, "INVOCATIONID": invocationid, "NETBIOSNAME": names.netbiosname, "DNSNAME": "%s.%s" % (names.hostname, names.dnsdomain), - "MACHINEPASS_B64": b64encode(machinepass.encode('utf-16-le')), + "MACHINEPASS_B64": b64encode(machinepass.encode('utf-16-le')).decode('utf8'), "DOMAINSID": str(domainsid), "DCRID": str(dc_rid), "SAMBA_VERSION_STRING": version, @@ -1171,7 +1171,7 @@ def setup_self_join(samdb, admin_session_info, names, fill, machinepass, "INVOCATIONID": invocationid, "NETBIOSNAME": names.netbiosname, "DNSNAME": "%s.%s" % (names.hostname, names.dnsdomain), - "MACHINEPASS_B64": b64encode(machinepass.encode('utf-16-le')), + "MACHINEPASS_B64": b64encode(machinepass.encode('utf-16-le')).decode('utf8'), "DOMAINSID": str(domainsid), "DCRID": str(dc_rid), "SAMBA_VERSION_STRING": version, @@ -1367,7 +1367,7 @@ def fill_samdb(samdb, lp, names, logger, policyguid, else: domainguid_line = "" - descr = b64encode(get_domain_descriptor(names.domainsid)) + descr = b64encode(get_domain_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision_basedn.ldif"), { "DOMAINDN": names.domaindn, "DOMAINSID": str(names.domainsid), @@ -1390,7 +1390,7 @@ def fill_samdb(samdb, lp, names, logger, policyguid, # If we are setting up a subdomain, then this has been replicated in, so we don't need to add it if fill == FILL_FULL: logger.info("Adding configuration container") - descr = b64encode(get_config_descriptor(names.domainsid)) + descr = b64encode(get_config_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision_configuration_basedn.ldif"), { "CONFIGDN": names.configdn, "DESCRIPTOR": descr, @@ -1420,12 +1420,12 @@ def fill_samdb(samdb, lp, names, logger, policyguid, if fill == FILL_FULL: logger.info("Setting up sam.ldb configuration data") - partitions_descr = b64encode(get_config_partitions_descriptor(names.domainsid)) - sites_descr = b64encode(get_config_sites_descriptor(names.domainsid)) - ntdsquotas_descr = b64encode(get_config_ntds_quotas_descriptor(names.domainsid)) - protected1_descr = b64encode(get_config_delete_protected1_descriptor(names.domainsid)) - protected1wd_descr = b64encode(get_config_delete_protected1wd_descriptor(names.domainsid)) - protected2_descr = b64encode(get_config_delete_protected2_descriptor(names.domainsid)) + partitions_descr = b64encode(get_config_partitions_descriptor(names.domainsid)).decode('utf8') + sites_descr = b64encode(get_config_sites_descriptor(names.domainsid)).decode('utf8') + ntdsquotas_descr = b64encode(get_config_ntds_quotas_descriptor(names.domainsid)).decode('utf8') + protected1_descr = b64encode(get_config_delete_protected1_descriptor(names.domainsid)).decode('utf8') + protected1wd_descr = b64encode(get_config_delete_protected1wd_descriptor(names.domainsid)).decode('utf8') + protected2_descr = b64encode(get_config_delete_protected2_descriptor(names.domainsid)).decode('utf8') if "2008" in schema.base_schema: # exclude 2012-specific changes if we're using a 2008 schema @@ -1475,7 +1475,7 @@ def fill_samdb(samdb, lp, names, logger, policyguid, }) logger.info("Adding users container") - users_desc = b64encode(get_domain_users_descriptor(names.domainsid)) + users_desc = b64encode(get_domain_users_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision_users_add.ldif"), { "DOMAINDN": names.domaindn, "USERS_DESCRIPTOR": users_desc @@ -1484,7 +1484,7 @@ def fill_samdb(samdb, lp, names, logger, policyguid, setup_modify_ldif(samdb, setup_path("provision_users_modify.ldif"), { "DOMAINDN": names.domaindn}) logger.info("Adding computers container") - computers_desc = b64encode(get_domain_computers_descriptor(names.domainsid)) + computers_desc = b64encode(get_domain_computers_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision_computers_add.ldif"), { "DOMAINDN": names.domaindn, "COMPUTERS_DESCRIPTOR": computers_desc @@ -1494,11 +1494,11 @@ def fill_samdb(samdb, lp, names, logger, policyguid, setup_path("provision_computers_modify.ldif"), { "DOMAINDN": names.domaindn}) logger.info("Setting up sam.ldb data") - infrastructure_desc = b64encode(get_domain_infrastructure_descriptor(names.domainsid)) - lostandfound_desc = b64encode(get_domain_delete_protected2_descriptor(names.domainsid)) - system_desc = b64encode(get_domain_delete_protected1_descriptor(names.domainsid)) - builtin_desc = b64encode(get_domain_builtin_descriptor(names.domainsid)) - controllers_desc = b64encode(get_domain_controllers_descriptor(names.domainsid)) + infrastructure_desc = b64encode(get_domain_infrastructure_descriptor(names.domainsid)).decode('utf8') + lostandfound_desc = b64encode(get_domain_delete_protected2_descriptor(names.domainsid)).decode('utf8') + system_desc = b64encode(get_domain_delete_protected1_descriptor(names.domainsid)).decode('utf8') + builtin_desc = b64encode(get_domain_builtin_descriptor(names.domainsid)).decode('utf8') + controllers_desc = b64encode(get_domain_controllers_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision.ldif"), { "CREATTIME": str(samba.unix2nttime(int(time.time()))), "DOMAINDN": names.domaindn, @@ -1517,14 +1517,14 @@ def fill_samdb(samdb, lp, names, logger, policyguid, # If we are setting up a subdomain, then this has been replicated in, so we don't need to add it if fill == FILL_FULL: - managedservice_descr = b64encode(get_managed_service_accounts_descriptor(names.domainsid)) + managedservice_descr = b64encode(get_managed_service_accounts_descriptor(names.domainsid)).decode('utf8') setup_modify_ldif(samdb, setup_path("provision_configuration_references.ldif"), { "CONFIGDN": names.configdn, "SCHEMADN": names.schemadn}) logger.info("Setting up well known security principals") - protected1wd_descr = b64encode(get_config_delete_protected1wd_descriptor(names.domainsid)) + protected1wd_descr = b64encode(get_config_delete_protected1wd_descriptor(names.domainsid)).decode('utf8') setup_add_ldif(samdb, setup_path("provision_well_known_sec_princ.ldif"), { "CONFIGDN": names.configdn, "WELLKNOWNPRINCIPALS_DESCRIPTOR": protected1wd_descr, @@ -1541,8 +1541,8 @@ def fill_samdb(samdb, lp, names, logger, policyguid, setup_add_ldif(samdb, setup_path("provision_users.ldif"), { "DOMAINDN": names.domaindn, "DOMAINSID": str(names.domainsid), - "ADMINPASS_B64": b64encode(adminpass.encode('utf-16-le')), - "KRBTGTPASS_B64": b64encode(krbtgtpass.encode('utf-16-le')) + "ADMINPASS_B64": b64encode(adminpass.encode('utf-16-le')).decode('utf8'), + "KRBTGTPASS_B64": b64encode(krbtgtpass.encode('utf-16-le')).decode('utf8') }, controls=["relax:0", "provision:0"]) logger.info("Setting up self join") @@ -1562,7 +1562,9 @@ def fill_samdb(samdb, lp, names, logger, policyguid, ntds_dn = "CN=NTDS Settings,%s" % names.serverdn names.ntdsguid = samdb.searchone(basedn=ntds_dn, attribute="objectGUID", expression="", scope=ldb.SCOPE_BASE) - assert isinstance(names.ntdsguid, str) + + names.ntdsguid = names.ntdsguid.decode('utf8') + assert isinstance(names.ntdsguid, string_types) return samdb @@ -1920,7 +1922,7 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths, msg = ldb.Message(ldb.Dn(samdb, samdb.searchone("distinguishedName", expression="samAccountName=%s$" % names.netbiosname, - scope=ldb.SCOPE_SUBTREE))) + scope=ldb.SCOPE_SUBTREE).decode('utf8'))) msg["msDS-SupportedEncryptionTypes"] = ldb.MessageElement( elements=kerberos_enctypes, flags=ldb.FLAG_MOD_REPLACE, name="msDS-SupportedEncryptionTypes") @@ -1938,8 +1940,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths, backend_store=backend_store) domainguid = samdb.searchone(basedn=samdb.get_default_basedn(), - attribute="objectGUID") - assert isinstance(domainguid, str) + attribute="objectGUID").decode('utf8') + assert isinstance(domainguid, string_types) lastProvisionUSNs = get_last_provision_usn(samdb) maxUSN = get_max_usn(samdb, str(names.rootdn)) @@ -2189,7 +2191,7 @@ def provision(logger, session_info, smbconf=None, if paths.sysvol and not os.path.exists(paths.sysvol): os.makedirs(paths.sysvol, 0o775) - ldapi_url = "ldapi://%s" % urllib.quote(paths.s4_ldapi_path, safe="") + ldapi_url = "ldapi://%s" % urllib_quote(paths.s4_ldapi_path, safe="") schema = Schema(domainsid, invocationid=invocationid, schemadn=names.schemadn, base_schema=base_schema) diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py index 3d59c701066..218f57fe679 100644 --- a/python/samba/provision/sambadns.py +++ b/python/samba/provision/sambadns.py @@ -247,12 +247,12 @@ def setup_dns_partitions(samdb, domainsid, domaindn, forestdn, configdn, setup_add_ldif(samdb, setup_path("provision_dnszones_partitions.ldif"), { "ZONE_DN": domainzone_dn, - "SECDESC" : b64encode(descriptor) + "SECDESC" : b64encode(descriptor).decode('utf8') }) if fill_level != FILL_SUBDOMAIN: setup_add_ldif(samdb, setup_path("provision_dnszones_partitions.ldif"), { "ZONE_DN": forestzone_dn, - "SECDESC" : b64encode(descriptor) + "SECDESC" : b64encode(descriptor).decode('utf8') }) domainzone_guid = get_domainguid(samdb, domainzone_dn) @@ -267,8 +267,8 @@ def setup_dns_partitions(samdb, domainsid, domaindn, forestdn, configdn, "ZONE_DNS": domainzone_dns, "CONFIGDN": configdn, "SERVERDN": serverdn, - "LOSTANDFOUND_DESCRIPTOR": b64encode(protected2_desc), - "INFRASTRUCTURE_DESCRIPTOR": b64encode(protected1_desc), + "LOSTANDFOUND_DESCRIPTOR": b64encode(protected2_desc).decode('utf8'), + "INFRASTRUCTURE_DESCRIPTOR": b64encode(protected1_desc).decode('utf8'), }) setup_modify_ldif(samdb, setup_path("provision_dnszones_modify.ldif"), { "CONFIGDN": configdn, @@ -287,8 +287,8 @@ def setup_dns_partitions(samdb, domainsid, domaindn, forestdn, configdn, "ZONE_DNS": forestzone_dns, "CONFIGDN": configdn, "SERVERDN": serverdn, - "LOSTANDFOUND_DESCRIPTOR": b64encode(protected2_desc), - "INFRASTRUCTURE_DESCRIPTOR": b64encode(protected1_desc), + "LOSTANDFOUND_DESCRIPTOR": b64encode(protected2_desc).decode('utf8'), + "INFRASTRUCTURE_DESCRIPTOR": b64encode(protected1_desc).decode('utf8'), }) setup_modify_ldif(samdb, setup_path("provision_dnszones_modify.ldif"), { "CONFIGDN": configdn, diff --git a/python/samba/samdb.py b/python/samba/samdb.py index b66afb7431c..a1a1f45b003 100644 --- a/python/samba/samdb.py +++ b/python/samba/samdb.py @@ -1024,12 +1024,12 @@ def sequence_number(self, seq_type): def get_dsServiceName(self): '''get the NTDS DN from the rootDSE''' res = self.search(base="", scope=ldb.SCOPE_BASE, attrs=["dsServiceName"]) - return res[0]["dsServiceName"][0] + return res[0]["dsServiceName"][0].decode('utf8') def get_serverName(self): '''get the server DN from the rootDSE''' res = self.search(base="", scope=ldb.SCOPE_BASE, attrs=["serverName"]) - return res[0]["serverName"][0] + return res[0]["serverName"][0].decode('utf8') def dns_lookup(self, dns_name, dns_partition=None): '''Do a DNS lookup in the database, returns the NDR database structures''' diff --git a/python/samba/schema.py b/python/samba/schema.py index 67ec357a285..e161aa8ca0b 100644 --- a/python/samba/schema.py +++ b/python/samba/schema.py @@ -122,7 +122,7 @@ def __init__(self, domain_sid, invocationid=None, schemadn=None, setup_path("provision_schema_basedn_modify.ldif"), {"SCHEMADN": schemadn, "OBJVERSION" : schema_version}) - descr = b64encode(get_schema_descriptor(domain_sid)) + descr = b64encode(get_schema_descriptor(domain_sid)).decode('utf8') self.schema_dn_add = read_and_sub_file( setup_path("provision_schema_basedn.ldif"), {"SCHEMADN": schemadn, "DESCRIPTOR": descr}) @@ -130,13 +130,14 @@ def __init__(self, domain_sid, invocationid=None, schemadn=None, if override_prefixmap is not None: self.prefixmap_data = override_prefixmap else: - self.prefixmap_data = open(setup_path("prefixMap.txt"), 'r').read() + self.prefixmap_data = open(setup_path("prefixMap.txt"), 'rb').read() if additional_prefixmap is not None: for map in additional_prefixmap: self.prefixmap_data += "%s\n" % map - self.prefixmap_data = b64encode(self.prefixmap_data) + #self.prefixmap_data = b64encode(self.prefixmap_data.encode('utf8')).decode('utf8') + self.prefixmap_data = b64encode(self.prefixmap_data).decode('utf8') # We don't actually add this ldif, just parse it prefixmap_ldif = "dn: %s\nprefixMap:: %s\n\n" % (self.schemadn, self.prefixmap_data) From fe81fa6c32f895cb96bce0e2c41bfe738d4994a4 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 10 Apr 2018 16:03:37 +0100 Subject: [PATCH 11/62] s4/selftest: enable samba.tests.dsdb_lock for python3 Signed-off-by: Noel Power --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index d1e5bc6a509..9d41e9eb77c 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -578,7 +578,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planoldpythontestsuite("none", "simple", extra_path=["%s/lib/tdb/python/tests" % srcdir()], name="tdb.python") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.sam", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dsdb", py3_compatible=True) -planpythontestsuite("none", "samba.tests.dsdb_lock") +planpythontestsuite("none", "samba.tests.dsdb_lock", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc") From 88283a275f65d066590934e89fca5e8529c58328 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 11:56:40 +0100 Subject: [PATCH 12/62] python/samba/tests: port samba.tests.unix for py3 Signed-off-by: Noel Power --- python/samba/tests/dcerpc/unix.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/python/samba/tests/dcerpc/unix.py b/python/samba/tests/dcerpc/unix.py index e8ef4da8630..ea128beca08 100644 --- a/python/samba/tests/dcerpc/unix.py +++ b/python/samba/tests/dcerpc/unix.py @@ -20,6 +20,9 @@ from samba.dcerpc import unixinfo from samba.tests import RpcInterfaceTestCase +from samba.compat import text_type as txt_type +from samba.compat import PY3 as is_py3 +import samba.compat class UnixinfoTests(RpcInterfaceTestCase): @@ -28,19 +31,21 @@ def setUp(self): self.conn = unixinfo.unixinfo("ncalrpc:", self.get_loadparm()) def test_getpwuid_int(self): - infos = self.conn.GetPWUid(range(512)) + infos = self.conn.GetPWUid(list(range(512))) self.assertEquals(512, len(infos)) self.assertEquals("/bin/false", infos[0].shell) - self.assertTrue(isinstance(infos[0].homedir, unicode)) + self.assertTrue(isinstance(infos[0].homedir, txt_type)) def test_getpwuid(self): - infos = self.conn.GetPWUid(map(long, range(512))) - self.assertEquals(512, len(infos)) - self.assertEquals("/bin/false", infos[0].shell) - self.assertTrue(isinstance(infos[0].homedir, unicode)) + # This test only relevant in PY2 + if not is_py3: + infos = self.conn.GetPWUid(map(long, range(512))) + self.assertEquals(512, len(infos)) + self.assertEquals("/bin/false", infos[0].shell) + self.assertTrue(isinstance(infos[0].homedir, txt_type)) def test_gidtosid(self): - self.conn.GidToSid(1000L) + self.conn.GidToSid(1000) def test_uidtosid(self): self.conn.UidToSid(1000) From f0f64fdbe37f0802dc722b2b04f1e954426da98b Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 16 Apr 2018 18:58:01 +0100 Subject: [PATCH 13/62] s4/selftest: enable samba.tests.dcerpc.unix for python3 Signed-off-by: Noel Power --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 9d41e9eb77c..de4bc2fb073 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -580,7 +580,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dsdb", py3_compatible=True) planpythontestsuite("none", "samba.tests.dsdb_lock", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join") From 2035f0b10757647528f4decd64a783502d36afdf Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 13:52:58 +0100 Subject: [PATCH 14/62] Add aliases for CStringIO.StringIO --- python/samba/compat.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/samba/compat.py b/python/samba/compat.py index d17dc3c2ce9..8771dae044f 100644 --- a/python/samba/compat.py +++ b/python/samba/compat.py @@ -30,6 +30,10 @@ integer_types = int, string_types = str text_type = str + + # alias + import io + StringIO = io.StringIO else: # compat functions from urllib import quote as urllib_quote @@ -39,3 +43,7 @@ integer_types = (int, long) string_types = basestring text_type = unicode + + # alias + import cStringIO + StringIO = cStringIO.StringIO From 52a4898c954e88305147fc067fd3b1c45797dbed Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 13:53:39 +0100 Subject: [PATCH 15/62] some Py2/Py3 compatability porting --- python/samba/netcmd/nettime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/netcmd/nettime.py b/python/samba/netcmd/nettime.py index 694b6adda9b..7c16aec92f3 100644 --- a/python/samba/netcmd/nettime.py +++ b/python/samba/netcmd/nettime.py @@ -17,7 +17,7 @@ # import samba.getopt as options -import common +from . import common from samba.net import Net from samba.netcmd import ( From 30071118b6bdf4ad977323b6f32b5808f4c55cfe Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 14:14:12 +0100 Subject: [PATCH 16/62] python/samba/tests: port CStringIO usage for py2/py3 compatability --- python/samba/tests/samba_tool/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/tests/samba_tool/base.py b/python/samba/tests/samba_tool/base.py index de257e3db09..9bffdabd229 100644 --- a/python/samba/tests/samba_tool/base.py +++ b/python/samba/tests/samba_tool/base.py @@ -25,7 +25,7 @@ import string from samba.auth import system_session from samba.samdb import SamDB -from cStringIO import StringIO +from samba.compat import StringIO from samba.netcmd.main import cmd_sambatool import samba.tests From 867972462444302aa28c49968be2f24d0a847a99 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 13:53:57 +0100 Subject: [PATCH 17/62] s4/selftest/tests: enable samba.tests.samba_tool.timecmd for python3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index de4bc2fb073..063b3965449 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -582,7 +582,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc") -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join") planpythontestsuite("none", "samba.tests.samba_tool.visualize") From 81f76986c5621e55bc1931ff9638ded1645fcf5f Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 15:17:36 +0100 Subject: [PATCH 18/62] python/samba: Bulk StringIO port for python3 --- python/samba/gpclass.py | 2 +- python/samba/tests/blackbox/samba_dnsupdate.py | 2 +- python/samba/tests/emulate/traffic.py | 2 +- python/samba/tests/netcmd.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/samba/gpclass.py b/python/samba/gpclass.py index 0966611b686..d9ffad2165f 100644 --- a/python/samba/gpclass.py +++ b/python/samba/gpclass.py @@ -21,7 +21,7 @@ sys.path.insert(0, "bin/python") from samba import NTSTATUSError from ConfigParser import ConfigParser -from StringIO import StringIO +from samba.compat import StringIO from abc import ABCMeta, abstractmethod import xml.etree.ElementTree as etree import re diff --git a/python/samba/tests/blackbox/samba_dnsupdate.py b/python/samba/tests/blackbox/samba_dnsupdate.py index e6cad3bbaba..1a659b0b423 100644 --- a/python/samba/tests/blackbox/samba_dnsupdate.py +++ b/python/samba/tests/blackbox/samba_dnsupdate.py @@ -17,7 +17,7 @@ # import samba.tests -from StringIO import StringIO +from samba.compat import StringIO from samba.netcmd.main import cmd_sambatool from samba.credentials import Credentials from samba.auth import system_session diff --git a/python/samba/tests/emulate/traffic.py b/python/samba/tests/emulate/traffic.py index a57c2f5bc4e..0cfae02bce0 100644 --- a/python/samba/tests/emulate/traffic.py +++ b/python/samba/tests/emulate/traffic.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # from pprint import pprint -from cStringIO import StringIO +from samba.compat import StringIO import samba.tests diff --git a/python/samba/tests/netcmd.py b/python/samba/tests/netcmd.py index 2867c031a17..6f5ea88eb99 100644 --- a/python/samba/tests/netcmd.py +++ b/python/samba/tests/netcmd.py @@ -17,7 +17,7 @@ """Tests for samba.netcmd.""" -from cStringIO import StringIO +from samba.compat import StringIO from samba.netcmd import Command from samba.netcmd.testparm import cmd_testparm from samba.netcmd.main import cmd_sambatool From 00a19f59f1adb9744e19b3f64d7d6e6c8a698436 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 16:48:03 +0100 Subject: [PATCH 19/62] python/samba: changes to make samba.tests.samba_tool.join run under py3 --- python/samba/join.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/samba/join.py b/python/samba/join.py index b5cab7e7cb3..fa8c1b37302 100644 --- a/python/samba/join.py +++ b/python/samba/join.py @@ -357,21 +357,21 @@ def get_behavior_version(ctx): def get_dnsHostName(ctx): res = ctx.samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=["dnsHostName"]) - return res[0]["dnsHostName"][0] + return res[0]["dnsHostName"][0].decode('utf8') def get_domain_name(ctx): '''get netbios name of the domain from the partitions record''' partitions_dn = ctx.samdb.get_partitions_dn() res = ctx.samdb.search(base=partitions_dn, scope=ldb.SCOPE_ONELEVEL, attrs=["nETBIOSName"], expression='ncName=%s' % ldb.binary_encode(str(ctx.samdb.get_default_basedn()))) - return res[0]["nETBIOSName"][0] + return res[0]["nETBIOSName"][0].decode('utf8') def get_forest_domain_name(ctx): '''get netbios name of the domain from the partitions record''' partitions_dn = ctx.samdb.get_partitions_dn() res = ctx.samdb.search(base=partitions_dn, scope=ldb.SCOPE_ONELEVEL, attrs=["nETBIOSName"], expression='ncName=%s' % ldb.binary_encode(str(ctx.samdb.get_root_basedn()))) - return res[0]["nETBIOSName"][0] + return res[0]["nETBIOSName"][0].decode('utf8') def get_parent_partition_dn(ctx): '''get the parent domain partition DN from parent DNS name''' From 3dc7fef438a4ea879feed16669321ea82b46c268 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 17 Apr 2018 16:48:56 +0100 Subject: [PATCH 20/62] s4/selftest: Enable samba.tests.samba_tool.join for py3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 063b3965449..d539870e4c1 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -583,7 +583,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join", py3_compatible=True) planpythontestsuite("none", "samba.tests.samba_tool.visualize") From 4eb518966c343cd0370c30c8a407259d78525c72 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 10:42:03 +0100 Subject: [PATCH 21/62] strings with utf content -> u (ensuring they are treated as unicode) --- python/samba/graph.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/samba/graph.py b/python/samba/graph.py index 6cdd2ef77ed..b8c5088bec3 100644 --- a/python/samba/graph.py +++ b/python/samba/graph.py @@ -202,8 +202,8 @@ def compile_graph_key(key_items, nodes_above=[], elisions=None, if short[0] == ',' and long[0] == ',': short = short[1:] long = long[1:] - elision_str += ('\nelision%d[shape=plaintext; style=solid; ' - 'label="\“%s” means “%s”\\r"]\n' + elision_str += (u'\nelision%d[shape=plaintext; style=solid; ' + u'label="\“%s” means “%s”\\r"]\n' % ((i, short, long))) above_lines = [] @@ -515,13 +515,13 @@ def distance_matrix(vertices, edges, write = lines.append if utf8: - vertical = '│' - horizontal = '─' - corner = '╭' + vertical = u'│' + horizontal = u'─' + corner = u'╭' #diagonal = '╲' - diagonal = '·' + diagonal = u'·' #missing = '🕱' - missing = '-' + missing = u'-' else: vertical, horizontal, corner, diagonal, missing = '|-,0-' From 999c070baf2200e34153009cbda2cd656ddee8ae Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 09:23:02 +0100 Subject: [PATCH 22/62] python/samba/tests: prefix expected with 'u' as graph funcs return unicode --- python/samba/tests/graph.py | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/python/samba/tests/graph.py b/python/samba/tests/graph.py index 7ea53caa734..38590e91e8d 100644 --- a/python/samba/tests/graph.py +++ b/python/samba/tests/graph.py @@ -124,7 +124,7 @@ def test_simple_distance(self): ('cat', 'dog')] expected = { - "utf8 True, colour None": ''' + "utf8 True, colour None": u''' destination ╭────── ant │╭───── bat @@ -138,7 +138,7 @@ def test_simple_distance(self): dog ---·-- elephant 2311·- frog 23121·''', - 'utf8 True, colour ansi': ''' + 'utf8 True, colour ansi': u''' destination ╭────── ant │╭───── bat @@ -153,7 +153,7 @@ def test_simple_distance(self): elephant 2311·-  frog 23121· ''', - 'utf8 True, colour ansi-heatmap': ''' + 'utf8 True, colour ansi-heatmap': u''' destination ╭────── ant │╭───── bat @@ -168,7 +168,7 @@ def test_simple_distance(self): elephant 2311·-  frog 23121· ''', - 'utf8 True, colour xterm-256color': ''' + 'utf8 True, colour xterm-256color': u''' destination ╭────── ant │╭───── bat @@ -183,7 +183,7 @@ def test_simple_distance(self): elephant 2311·-  frog 23121· ''', -'utf8 True, colour xterm-256color-heatmap': ''' +'utf8 True, colour xterm-256color-heatmap': u''' destination ╭────── ant │╭───── bat @@ -198,7 +198,7 @@ def test_simple_distance(self): elephant 2311·-  frog 23121· ''', -'utf8 False, colour None': ''' +'utf8 False, colour None': u''' destination ,------ ant |,----- bat @@ -213,7 +213,7 @@ def test_simple_distance(self): elephant 23110- frog 231210 ''', -'utf8 False, colour ansi': ''' +'utf8 False, colour ansi': u''' destination ,------ ant |,----- bat @@ -228,7 +228,7 @@ def test_simple_distance(self): elephant 23110-  frog 231210 ''', -'utf8 False, colour ansi-heatmap': ''' +'utf8 False, colour ansi-heatmap': u''' destination ,------ ant |,----- bat @@ -243,7 +243,7 @@ def test_simple_distance(self): elephant 23110-  frog 231210 ''', -'utf8 False, colour xterm-256color': ''' +'utf8 False, colour xterm-256color': u''' destination ,------ ant |,----- bat @@ -258,7 +258,7 @@ def test_simple_distance(self): elephant 23110-  frog 231210 ''', -'utf8 False, colour xterm-256color-heatmap': ''' +'utf8 False, colour xterm-256color-heatmap': u''' destination ,------ ant |,----- bat @@ -288,7 +288,7 @@ def test_simple_distance2(self): ('bat', 'ant'), ('ant', 'cat')] expected = { - 'utf8 True, colour None': ''' + 'utf8 True, colour None': u''' destination ╭─── ant │╭── bat @@ -297,7 +297,7 @@ def test_simple_distance2(self): bat 1·2 cat 21· ''', - 'utf8 True, colour ansi': ''' + 'utf8 True, colour ansi': u''' destination ╭─── ant │╭── bat @@ -306,7 +306,7 @@ def test_simple_distance2(self):  bat 1·2  cat 21· ''', -'utf8 True, colour ansi-heatmap': ''' +'utf8 True, colour ansi-heatmap': u''' destination ╭─── ant │╭── bat @@ -315,7 +315,7 @@ def test_simple_distance2(self):  bat 1·2  cat 21· ''', -'utf8 True, colour xterm-256color': ''' +'utf8 True, colour xterm-256color': u''' destination ╭─── ant │╭── bat @@ -324,7 +324,7 @@ def test_simple_distance2(self):  bat 1·2  cat 21· ''', -'utf8 True, colour xterm-256color-heatmap': ''' +'utf8 True, colour xterm-256color-heatmap': u''' destination ╭─── ant │╭── bat @@ -333,7 +333,7 @@ def test_simple_distance2(self):  bat 1·2  cat 21· ''', -'utf8 False, colour None': ''' +'utf8 False, colour None': u''' destination ,--- ant |,-- bat @@ -342,7 +342,7 @@ def test_simple_distance2(self): bat 102 cat 210 ''', -'utf8 False, colour ansi': ''' +'utf8 False, colour ansi': u''' destination ,--- ant |,-- bat @@ -351,7 +351,7 @@ def test_simple_distance2(self):  bat 102  cat 210 ''', -'utf8 False, colour ansi-heatmap': ''' +'utf8 False, colour ansi-heatmap': u''' destination ,--- ant |,-- bat @@ -360,7 +360,7 @@ def test_simple_distance2(self):  bat 102  cat 210 ''', -'utf8 False, colour xterm-256color': ''' +'utf8 False, colour xterm-256color': u''' destination ,--- ant |,-- bat @@ -369,7 +369,7 @@ def test_simple_distance2(self):  bat 102  cat 210 ''', -'utf8 False, colour xterm-256color-heatmap': ''' +'utf8 False, colour xterm-256color-heatmap': u''' destination ,--- ant |,-- bat @@ -394,7 +394,7 @@ def test_simple_distance3(self): ('dog', 'ant'), ('dog', 'eel')] expected = { -'utf8 True, colour None': ''' +'utf8 True, colour None': u''' destination ╭───── ant │╭──── bat @@ -407,7 +407,7 @@ def test_simple_distance3(self): dog 123·1 eel ----· ''', -'utf8 True, colour ansi': ''' +'utf8 True, colour ansi': u''' destination ╭───── ant │╭──── bat @@ -420,7 +420,7 @@ def test_simple_distance3(self):  dog 123·1  eel ----· ''', -'utf8 True, colour ansi-heatmap': ''' +'utf8 True, colour ansi-heatmap': u''' destination ╭───── ant │╭──── bat @@ -433,7 +433,7 @@ def test_simple_distance3(self):  dog 123·1  eel ----· ''', -'utf8 True, colour xterm-256color': ''' +'utf8 True, colour xterm-256color': u''' destination ╭───── ant │╭──── bat @@ -446,7 +446,7 @@ def test_simple_distance3(self):  dog 123·1  eel ----· ''', -'utf8 True, colour xterm-256color-heatmap': ''' +'utf8 True, colour xterm-256color-heatmap': u''' destination ╭───── ant │╭──── bat @@ -459,7 +459,7 @@ def test_simple_distance3(self):  dog 123·1  eel ----· ''', -'utf8 False, colour None': ''' +'utf8 False, colour None': u''' destination ,----- ant |,---- bat @@ -472,7 +472,7 @@ def test_simple_distance3(self): dog 12301 eel ----0 ''', -'utf8 False, colour ansi': ''' +'utf8 False, colour ansi': u''' destination ,----- ant |,---- bat @@ -485,7 +485,7 @@ def test_simple_distance3(self):  dog 12301  eel ----0 ''', -'utf8 False, colour ansi-heatmap': ''' +'utf8 False, colour ansi-heatmap': u''' destination ,----- ant |,---- bat @@ -499,7 +499,7 @@ def test_simple_distance3(self):  eel ----0 ''', 'utf8 False, colour xterm-256color': -''' destination +u''' destination ,----- ant |,---- bat ||,--- cat @@ -511,7 +511,7 @@ def test_simple_distance3(self):  dog 12301  eel ----0 ''', -'utf8 False, colour xterm-256color-heatmap': ''' +'utf8 False, colour xterm-256color-heatmap': u''' destination ,----- ant |,---- bat From e83184b7db352f955d7b732a11e24f2ac169c106 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 12:33:51 +0100 Subject: [PATCH 23/62] targeted p3 port use io.open to be able to specifiy encoding for text we write specify utf8 in write (hmmm maybe we should just be doing this anyway regardless) #TODO check again --- python/samba/netcmd/visualize.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index 6f880ae61e8..d5f2c205fb0 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Visualisation tools # # Copyright (C) Andrew Bartlett 2015, 2018 @@ -34,7 +35,7 @@ import time from samba.kcc import KCC from samba.kcc.kcc_utils import KCCError - +from io import open COMMON_OPTIONS = [ Option("-H", "--URL", help="LDB URL for database or target server", type=str, metavar="URL", dest="H"), @@ -94,7 +95,7 @@ def get_kcc_and_dsas(self, H, lp, creds): return kcc, dsas - def write(self, s, fn=None, suffix='.dot'): + def write(self, s, fn=None, suffix='.dot', utf8=None): """Decide whether we're dealing with a filename, a tempfile, or stdout, and write accordingly. @@ -106,18 +107,20 @@ def write(self, s, fn=None, suffix='.dot'): If fn is visualize.TEMP_FILE, write to a temporary file Otherwise fn should be a filename to write to. """ + encoding = None; + if utf8: + encoding = 'utf8'; if fn is None or fn == '-': # we're just using stdout (a.k.a self.outf) print(s, file=self.outf) return - if fn is TEMP_FILE: fd, fn = tempfile.mkstemp(prefix='samba-tool-visualise', suffix=suffix) - f = open(fn, 'w') + f = open(fn, 'w', encoding=encoding) os.close(fd) else: - f = open(fn, 'w') + f = open(fn, 'w', encoding=encoding) f.write(s) f.close() @@ -315,7 +318,7 @@ def run(self, H=None, output=None, shorten_names=False, generate_key=key) s = "\n%s\n%s" % (header_strings[direction] % part, s) - self.write(s, output) + self.write(s, output, utf8=utf8) return edge_colours = [] @@ -361,7 +364,7 @@ def run(self, H=None, output=None, shorten_names=False, shorten_names=shorten_names, key_items=key_items) - self.write(s, output) + self.write(s, output, utf8=utf8) class NTDSConn(object): @@ -509,7 +512,7 @@ def run(self, H=None, output=None, shorten_names=False, colour=color_scheme, shorten_names=shorten_names, generate_key=key) - self.write('\n%s\n%s\n%s' % (title, s, epilog), output) + self.write('\n%s\n%s\n%s' % (title, s, epilog), output, utf8=utf8) return dot_edges = [] @@ -566,7 +569,7 @@ def run(self, H=None, output=None, shorten_names=False, edge_styles=edge_styles, shorten_names=shorten_names, key_items=key_items) - self.write(s, output) + self.write(s, output, utf8=utf8) class cmd_visualize(SuperCommand): From 704e0d8f78ffea6e9a86fe5d4fc6e9b454a11f55 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 14:20:57 +0100 Subject: [PATCH 24/62] python/samba: use StringIO (not cStringIO) for python2 cStringIO doesn't handle unicode, StringIO does. With py2/py3 compatable code we can easily find ourselves getting passed unicode --- python/samba/compat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/compat.py b/python/samba/compat.py index 8771dae044f..5662095fed5 100644 --- a/python/samba/compat.py +++ b/python/samba/compat.py @@ -45,5 +45,5 @@ text_type = unicode # alias - import cStringIO - StringIO = cStringIO.StringIO + import StringIO + StringIO = StringIO.StringIO From ebbe9d170b86fe82c641ceb2b6eed0ec9e4e8a8e Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 16:51:21 +0100 Subject: [PATCH 25/62] make sure only unicode is written to 'write' --- python/samba/netcmd/visualize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index d5f2c205fb0..0096380429d 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -121,7 +121,8 @@ def write(self, s, fn=None, suffix='.dot', utf8=None): os.close(fd) else: f = open(fn, 'w', encoding=encoding) - + if isinstance(s, str): + s = s.decode('utf8') f.write(s) f.close() return fn From 014ef0fd534cb8347672aec5bdfae730ae2db1e1 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 17:56:53 +0100 Subject: [PATCH 26/62] python/samba/tests: more tweaks to get samba.tests.visualize to run Note: this is NOT to get the visualize tests to run via py2/py3 but rather get the tests to run post some changes that change the result of some functions e.g. get_dsServiceName() & get_serverName() in python/samba/samdb.py which now return unicode strings which was needed to ensure py2/py3 compatability when running samba.tests.dsdb_lock under py2/py3 --- python/samba/tests/samba_tool/visualize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/samba/tests/samba_tool/visualize.py b/python/samba/tests/samba_tool/visualize.py index a90c77364b7..25c56cc6ec6 100644 --- a/python/samba/tests/samba_tool/visualize.py +++ b/python/samba/tests/samba_tool/visualize.py @@ -200,7 +200,7 @@ def test_utf8(self): '-H', self.dburl, '--color=no', '-S') self.assertCmdSuccess(result, ascii, err) - for c in ('│', '─', '╭'): + for c in (u'│', u'─', u'╭'): self.assertTrue(c in utf8, 'UTF8 should contain %s' % c) self.assertTrue(c not in ascii, 'ASCII should not contain %s' % c) @@ -319,9 +319,9 @@ def test_dot_ntdsconn_disconnected(self): '-o', '-') self.assertCmdSuccess(result, dot, err) self.remove_files(dbfile) - print(dot) + print(dot.encode('utf8')) - self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot, + self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot.encode('utf8'), strip=True) def test_dot_ntdsconn_disconnected_to_file(self): From f7cba3ae5af40591fa47525f1583bd1f79de008c Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 18 Apr 2018 19:44:23 +0100 Subject: [PATCH 27/62] tweak test and impl a little more to try and get it to run under py2 still py2 (but using unicode) --- python/samba/netcmd/visualize.py | 8 ++++---- python/samba/tests/samba_tool/visualize.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index 0096380429d..9c0383f1f89 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -107,9 +107,11 @@ def write(self, s, fn=None, suffix='.dot', utf8=None): If fn is visualize.TEMP_FILE, write to a temporary file Otherwise fn should be a filename to write to. """ - encoding = None; + encoding = None if utf8: - encoding = 'utf8'; + encoding = 'utf8' + if isinstance(s, str): + s = s.decode('utf8') if fn is None or fn == '-': # we're just using stdout (a.k.a self.outf) print(s, file=self.outf) @@ -121,8 +123,6 @@ def write(self, s, fn=None, suffix='.dot', utf8=None): os.close(fd) else: f = open(fn, 'w', encoding=encoding) - if isinstance(s, str): - s = s.decode('utf8') f.write(s) f.close() return fn diff --git a/python/samba/tests/samba_tool/visualize.py b/python/samba/tests/samba_tool/visualize.py index 25c56cc6ec6..519ba3c9081 100644 --- a/python/samba/tests/samba_tool/visualize.py +++ b/python/samba/tests/samba_tool/visualize.py @@ -337,13 +337,14 @@ def test_dot_ntdsconn_disconnected_to_file(self): '--color=no', '-S', '--dot', '-o', dot_file) self.assertCmdSuccess(result, dot, err) + from io import open f = open(dot_file) dot = f.read() f.close() - self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot) + self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot.encode('utf8')) self.remove_files(dbfile, dot_file) - print(dot) + print(dot.encode('utf8')) EXPECTED_DOT_MULTISITE_NO_KEY = r"""/* generated by samba */ digraph A_samba_tool_production { From a89438f36c16d06f6693a701e2930c64c72c852c Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 19 Apr 2018 16:05:14 +0100 Subject: [PATCH 28/62] python2 unicode support changes. Bit of a descrepency, it seems we write unicode when writing file but return when no output file or '-' is specified. Adjust test so that we don't use r"blah blah blah" raw string format as this will screw up when we use python3. python 2 dot type from file is unicode raw string is python3 dot type from file is raw string is This means for the test to pass running python2 we need to encode the return from the file (e.g. unicde) to convert to so we can compare. With python3 we can't do that (no encode method for ). We need the result and expected result types to match (in python2 & python3) --- python/samba/tests/samba_tool/visualize.py | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/python/samba/tests/samba_tool/visualize.py b/python/samba/tests/samba_tool/visualize.py index 519ba3c9081..502bb8fda67 100644 --- a/python/samba/tests/samba_tool/visualize.py +++ b/python/samba/tests/samba_tool/visualize.py @@ -321,7 +321,7 @@ def test_dot_ntdsconn_disconnected(self): self.remove_files(dbfile) print(dot.encode('utf8')) - self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot.encode('utf8'), + self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot, strip=True) def test_dot_ntdsconn_disconnected_to_file(self): @@ -341,7 +341,7 @@ def test_dot_ntdsconn_disconnected_to_file(self): f = open(dot_file) dot = f.read() f.close() - self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot.encode('utf8')) + self.assertStringsEqual(EXPECTED_DOT_NTDSCONN_DISCONNECTED, dot) self.remove_files(dbfile, dot_file) print(dot.encode('utf8')) @@ -386,25 +386,25 @@ def test_dot_ntdsconn_disconnected_to_file(self): """ -EXPECTED_DOT_NTDSCONN_DISCONNECTED = r"""/* generated by samba */ +EXPECTED_DOT_NTDSCONN_DISCONNECTED = u"""/* generated by samba */ digraph A_samba_tool_production { label="NTDS Connections known to CN=LOCALDC,CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com"; fontsize=10; node[fontname=Helvetica; fontsize=10]; -"CN=NTDS Settings,\nCN=CLIENT,\n..."; -"CN=NTDS Settings,\nCN=LOCALDC,\n..."; -"CN=NTDS Settings,\nCN=PROMOTEDVDC,\n..."; -"CN=NTDS Settings,\nCN=SERVER1,\n..."; -"CN=NTDS Settings,\nCN=SERVER2,\n..."; -"CN=NTDS Settings,\nCN=SERVER3,\n..."; -"CN=NTDS Settings,\nCN=SERVER4,\n..."; -"CN=NTDS Settings,\nCN=SERVER5,\n..."; -"CN=NTDS Settings,\nCN=LOCALDC,\n..." -> "CN=NTDS Settings,\nCN=PROMOTEDVDC,\n..." [color="#000000", ]; -"CN=NTDS Settings,\nCN=PROMOTEDVDC,\n..." -> "CN=NTDS Settings,\nCN=LOCALDC,\n..." [color="#000000", ]; -"CN=NTDS Settings,\nCN=SERVER2,\n..." -> "CN=NTDS Settings,\nCN=PROMOTEDVDC,\n..." [color="#000000", ]; -"CN=NTDS Settings,\nCN=SERVER3,\n..." -> "CN=NTDS Settings,\nCN=LOCALDC,\n..." [color="#000000", ]; +"CN=NTDS Settings,\\nCN=CLIENT,\\n..."; +"CN=NTDS Settings,\\nCN=LOCALDC,\\n..."; +"CN=NTDS Settings,\\nCN=PROMOTEDVDC,\\n..."; +"CN=NTDS Settings,\\nCN=SERVER1,\\n..."; +"CN=NTDS Settings,\\nCN=SERVER2,\\n..."; +"CN=NTDS Settings,\\nCN=SERVER3,\\n..."; +"CN=NTDS Settings,\\nCN=SERVER4,\\n..."; +"CN=NTDS Settings,\\nCN=SERVER5,\\n..."; +"CN=NTDS Settings,\\nCN=LOCALDC,\\n..." -> "CN=NTDS Settings,\\nCN=PROMOTEDVDC,\\n..." [color="#000000", ]; +"CN=NTDS Settings,\\nCN=PROMOTEDVDC,\\n..." -> "CN=NTDS Settings,\\nCN=LOCALDC,\\n..." [color="#000000", ]; +"CN=NTDS Settings,\\nCN=SERVER2,\\n..." -> "CN=NTDS Settings,\\nCN=PROMOTEDVDC,\\n..." [color="#000000", ]; +"CN=NTDS Settings,\\nCN=SERVER3,\\n..." -> "CN=NTDS Settings,\\nCN=LOCALDC,\\n..." [color="#000000", ]; subgraph cluster_key { label="Key"; subgraph cluster_key_nodes { @@ -419,22 +419,22 @@ def test_dot_ntdsconn_disconnected_to_file(self): key_0_e1[label=src; color="#000000"; group="key_0__g"] key_0_e2[label=dest; color="#000000"; group="key_0__g"] key_0_e1 -> key_0_e2 [constraint = false; color="#000000"] -key_0__label[shape=plaintext; style=solid; width=2.000000; label="NTDS Connection\r"] +key_0__label[shape=plaintext; style=solid; width=2.000000; label="NTDS Connection\\r"] } {key_0__label} } -elision0[shape=plaintext; style=solid; label="\“...” means “CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com”\r"] +elision0[shape=plaintext; style=solid; label="\\“...” means “CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=samba,DC=example,DC=com”\\r"] } -"CN=NTDS Settings,\nCN=CLIENT,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=LOCALDC,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=PROMOTEDVDC,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=SERVER1,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=SERVER2,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=SERVER3,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=SERVER4,\n..." -> key_0__label [style=invis]; -"CN=NTDS Settings,\nCN=SERVER5,\n..." -> key_0__label [style=invis] +"CN=NTDS Settings,\\nCN=CLIENT,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=LOCALDC,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=PROMOTEDVDC,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=SERVER1,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=SERVER2,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=SERVER3,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=SERVER4,\\n..." -> key_0__label [style=invis]; +"CN=NTDS Settings,\\nCN=SERVER5,\\n..." -> key_0__label [style=invis] key_0__label -> elision0 [style=invis; weight=9] } From d50256887d8131d89939273eaf8820adf7cc2e21 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 19 Apr 2018 16:15:58 +0100 Subject: [PATCH 29/62] python3: Ensure some attributes are coverted to string before use. --- python/samba/kcc/__init__.py | 2 +- python/samba/kcc/kcc_utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/samba/kcc/__init__.py b/python/samba/kcc/__init__.py index 1a962bb03c1..be1c4d82968 100644 --- a/python/samba/kcc/__init__.py +++ b/python/samba/kcc/__init__.py @@ -274,7 +274,7 @@ def load_my_dsa(self): scope=ldb.SCOPE_BASE, attrs=["dsServiceName"]) dn = ldb.Dn(self.samdb, - service_name_res[0]["dsServiceName"][0]) + service_name_res[0]["dsServiceName"][0].decode('utf8')) res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE, attrs=["objectGUID"]) diff --git a/python/samba/kcc/kcc_utils.py b/python/samba/kcc/kcc_utils.py index 2118570bbfc..aac2abfc15c 100644 --- a/python/samba/kcc/kcc_utils.py +++ b/python/samba/kcc/kcc_utils.py @@ -993,7 +993,7 @@ def load_connection(self, samdb): self.schedule = ndr_unpack(drsblobs.schedule, msg["schedule"][0]) if "whenCreated" in msg: - self.whenCreated = ldb.string_to_time(msg["whenCreated"][0]) + self.whenCreated = ldb.string_to_time(msg["whenCreated"][0].decode('utf8')) if "fromServer" in msg: dsdn = dsdb_Dn(samdb, msg["fromServer"][0]) @@ -1919,10 +1919,10 @@ def load_transport(self, samdb): self.options = int(msg["options"][0]) if "transportAddressAttribute" in msg: - self.address_attr = str(msg["transportAddressAttribute"][0]) + self.address_attr = msg["transportAddressAttribute"][0].decode('utf8') if "name" in msg: - self.name = str(msg["name"][0]) + self.name = msg["name"][0].decode('utf8') if "bridgeheadServerListBL" in msg: for value in msg["bridgeheadServerListBL"]: From 754e3b6699a70f5af26a8b8e29548e80e8b1825a Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 19 Apr 2018 16:17:02 +0100 Subject: [PATCH 30/62] python3: couple of changes to ensure visualize works with py2/py3 * convert keys to list (is an iterator in py3) and since we are passing it as a param someone could try to find an item in it more than once (and that would fail) * make sure a bytes attribute is converted to string * make sure we handles a bytes in py3 or str in py2 and convert to str and unicode respectively --- python/samba/netcmd/visualize.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index 9c0383f1f89..ad684e92413 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -35,6 +35,7 @@ import time from samba.kcc import KCC from samba.kcc.kcc_utils import KCCError +from samba.compat import text_type from io import open COMMON_OPTIONS = [ Option("-H", "--URL", help="LDB URL for database or target server", @@ -51,7 +52,7 @@ choices=['yes', 'no', 'auto']), Option("--color-scheme", help=("use this colour scheme " "(implies --color=yes)"), - choices=COLOUR_SETS.keys()), + choices=list(COLOUR_SETS.keys())), Option("-S", "--shorten-names", help="don't print long common suffixes", action='store_true', default=False), @@ -110,7 +111,7 @@ def write(self, s, fn=None, suffix='.dot', utf8=None): encoding = None if utf8: encoding = 'utf8' - if isinstance(s, str): + if not isinstance(s, text_type): s = s.decode('utf8') if fn is None or fn == '-': # we're just using stdout (a.k.a self.outf) @@ -434,7 +435,7 @@ def run(self, H=None, output=None, shorten_names=False, for msg in res: msgdn = str(msg.dn) dest_dn = msgdn[msgdn.index(',') + 1:] - attested_edges.append((msg['fromServer'][0], + attested_edges.append((msg['fromServer'][0].decode('utf8'), dest_dn, ntds_dn)) # now we overlay all the graphs and generate styles accordingly @@ -455,7 +456,7 @@ def run(self, H=None, output=None, shorten_names=False, if not talk_to_remote: # If we are not talking to remote servers, we list all # the connections. - graph_edges = edges.keys() + graph_edges = list(edges.keys()) title = 'NTDS Connections known to %s' % local_dsa_dn epilog = '' From 87ba1d1039f5e313f04ec27c0810bd9fcc0dc84e Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 21:14:41 +0100 Subject: [PATCH 31/62] python/samba/kcc: Py2/Py3 compat, ensure 2nd param for dsdb_Dn is string --- python/samba/kcc/kcc_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/samba/kcc/kcc_utils.py b/python/samba/kcc/kcc_utils.py index aac2abfc15c..adc9d02c93c 100644 --- a/python/samba/kcc/kcc_utils.py +++ b/python/samba/kcc/kcc_utils.py @@ -739,7 +739,7 @@ def load_current_replica_table(self, samdb): for value in res[0][k]: # Turn dn into a dsdb_Dn so we can use # its methods to parse a binary DN - dsdn = dsdb_Dn(samdb, value) + dsdn = dsdb_Dn(samdb, value.decode('utf8')) flags = dsdn.get_binary_integer() dnstr = str(dsdn.dn) @@ -986,7 +986,7 @@ def load_connection(self, samdb): "for (%s)" % (self.dnstr)) if "transportType" in msg: - dsdn = dsdb_Dn(samdb, msg["transportType"][0]) + dsdn = dsdb_Dn(samdb, msg["transportType"][0].decode('utf8')) self.load_connection_transport(samdb, str(dsdn.dn)) if "schedule" in msg: @@ -996,7 +996,7 @@ def load_connection(self, samdb): self.whenCreated = ldb.string_to_time(msg["whenCreated"][0].decode('utf8')) if "fromServer" in msg: - dsdn = dsdb_Dn(samdb, msg["fromServer"][0]) + dsdn = dsdb_Dn(samdb, msg["fromServer"][0].decode('utf8')) self.from_dnstr = str(dsdn.dn) assert self.from_dnstr is not None @@ -1363,7 +1363,7 @@ def load_partition(self, samdb): continue for value in msg[k]: - dsdn = dsdb_Dn(samdb, value) + dsdn = dsdb_Dn(samdb, value.decode('utf8')) dnstr = str(dsdn.dn) if k == "nCName": @@ -1926,7 +1926,7 @@ def load_transport(self, samdb): if "bridgeheadServerListBL" in msg: for value in msg["bridgeheadServerListBL"]: - dsdn = dsdb_Dn(samdb, value) + dsdn = dsdb_Dn(samdb, value.decode('utf8')) dnstr = str(dsdn.dn) if dnstr not in self.bridgehead_list: self.bridgehead_list.append(dnstr) @@ -2188,7 +2188,7 @@ def load_sitelink(self, samdb): if "siteList" in msg: for value in msg["siteList"]: - dsdn = dsdb_Dn(samdb, value) + dsdn = dsdb_Dn(samdb, value.decode('utf8')) guid = misc.GUID(dsdn.dn.get_extended_component('GUID')) if guid not in self.site_list: self.site_list.append(guid) From 0f29c89764b76f159cdb4bbe55e96a8dd9c1ba9c Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 20 Apr 2018 16:34:17 +0100 Subject: [PATCH 32/62] python2/python3 compat tweaks --- python/samba/netcmd/visualize.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py index ad684e92413..3a4c092c21f 100644 --- a/python/samba/netcmd/visualize.py +++ b/python/samba/netcmd/visualize.py @@ -166,7 +166,10 @@ def colour_hash(x): """Generate a randomish but consistent darkish colour based on the given object.""" from hashlib import md5 - c = int(md5(str(x)).hexdigest()[:6], base=16) & 0x7f7f7f + tmp_str = str(x) + if isinstance(tmp_str, text_type): + tmp_str = tmp_str.encode('utf8') + c = int(md5(tmp_str).hexdigest()[:6], base=16) & 0x7f7f7f return '#%06x' % c @@ -405,7 +408,7 @@ def run(self, H=None, output=None, shorten_names=False, res = local_kcc.samdb.search(dsa_dn, scope=SCOPE_BASE, attrs=["dNSHostName"]) - dns_name = res[0]["dNSHostName"][0] + dns_name = res[0]["dNSHostName"][0].decode('utf8') try: samdb = self.get_db("ldap://%s" % dns_name, sambaopts, credopts) From a0b4c354c98d33295bad059d2ee86e399eb48a59 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 20:34:38 +0100 Subject: [PATCH 33/62] python/samba/tests/samba_tool: fix a native string containing some non ascii "HIGH\xFFBYTE" is a string containing such text. In python2 we seem to get some problems when intermixing 'str' (e.g. bytes) with proper unicode. Note: some of the py2/py3 changes have peppered unicode around the place (where it wasn't before). --- python/samba/tests/samba_tool/dnscmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/tests/samba_tool/dnscmd.py b/python/samba/tests/samba_tool/dnscmd.py index c44a3083716..2c3ced7d03c 100644 --- a/python/samba/tests/samba_tool/dnscmd.py +++ b/python/samba/tests/samba_tool/dnscmd.py @@ -48,7 +48,7 @@ def setUp(self): "EXAMPLE", "\n.COM", "!@#$%^&*()_", - "HIGH\xFFBYTE", + u"HIGH\xFFBYTE", "@.EXAMPLE.COM", "."] bad_dns = ["...", From 4f08387bf365381d059b07709a8f7baaeba85a65 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 20 Apr 2018 17:24:44 +0100 Subject: [PATCH 34/62] enable samba_tool.visualize_drs samba_tool.visualize_drs for py3 --- source4/selftest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index d539870e4c1..f5d5d95e7e1 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -585,7 +585,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join", py3_compatible=True) -planpythontestsuite("none", "samba.tests.samba_tool.visualize") +planpythontestsuite("none", "samba.tests.samba_tool.visualize", py3_compatible=True) # test fsmo show @@ -1016,7 +1016,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex }, extra_path=[os.path.join(srcdir(), "samba/python"), ] ) - planpythontestsuite(env, "samba.tests.samba_tool.visualize_drs") + planpythontestsuite(env, "samba.tests.samba_tool.visualize_drs", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.kcc.kcc_utils") From fed602e33a05e476bff803f5345e96bf4dadfebe Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 20 Apr 2018 17:42:41 +0100 Subject: [PATCH 35/62] enable samba.tests.dcerpc.srvsvc (even though it seemsnot run (dangerous test) --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index f5d5d95e7e1..80e9b14a619 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -581,7 +581,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("none", "samba.tests.dsdb_lock", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.bare", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.unix", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.dcerpc.srvsvc", py3_compatible=False) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.timecmd", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.join", py3_compatible=True) From 521720e5cb782f280c07dac41dd814e404bf4253 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 12:21:31 +0100 Subject: [PATCH 36/62] python/samba/tests: Port fsmo test to python3 --- python/samba/tests/samba_tool/fsmo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/tests/samba_tool/fsmo.py b/python/samba/tests/samba_tool/fsmo.py index 7fd0ae2e033..847239d862d 100644 --- a/python/samba/tests/samba_tool/fsmo.py +++ b/python/samba/tests/samba_tool/fsmo.py @@ -36,7 +36,7 @@ def test_fsmoget(self): res = samdb.search(base=ldb.Dn(samdb, "CN=Infrastructure,DC=DomainDnsZones") + samdb.get_default_basedn(), scope=ldb.SCOPE_BASE, attrs=["fsmoRoleOwner"]) - self.assertTrue("DomainDnsZonesMasterRole owner: " + res[0]["fsmoRoleOwner"][0] in out) + self.assertTrue("DomainDnsZonesMasterRole owner: " + res[0]["fsmoRoleOwner"][0].decode('utf8') in out) except ldb.LdbError as e: (enum, string) = e.args if enum == ldb.ERR_NO_SUCH_OBJECT: @@ -47,4 +47,4 @@ def test_fsmoget(self): res = samdb.search(base=samdb.get_default_basedn(), scope=ldb.SCOPE_BASE, attrs=["fsmoRoleOwner"]) - self.assertTrue("DomainNamingMasterRole owner: " + res[0]["fsmoRoleOwner"][0] in out) + self.assertTrue("DomainNamingMasterRole owner: " + res[0]["fsmoRoleOwner"][0].decode('utf8') in out) From 214e3eb03d3d619731951fb9578029e0c714a856 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 12:21:53 +0100 Subject: [PATCH 37/62] s3/selftest: Enable samba.tests.samba_tool.fsmo for python3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 80e9b14a619..577e325b635 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -590,7 +590,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex # test fsmo show for env in ["ad_dc_ntvfs", "fl2000dc", "fl2003dc", "fl2008r2dc"]: - planpythontestsuite(env + ":local", "samba.tests.samba_tool.fsmo") + planpythontestsuite(env + ":local", "samba.tests.samba_tool.fsmo", py3_compatible=True) # test user.edit for env in ["ad_dc:local", "ad_dc_ntvfs:local", "fl2000dc:local", "fl2003dc:local", "fl2008r2dc:local"]: From e615555a33dddd1be0e864f95f2c7163fb57cc7f Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 14:01:01 +0100 Subject: [PATCH 38/62] python/samba/netcmd: some changes to be python2/python3 compatable --- python/samba/netcmd/gpo.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/samba/netcmd/gpo.py b/python/samba/netcmd/gpo.py index cd946d9fa72..965dd3960d3 100644 --- a/python/samba/netcmd/gpo.py +++ b/python/samba/netcmd/gpo.py @@ -252,7 +252,7 @@ def copy_directory_remote_to_local(conn, remotedir, localdir): os.mkdir(l_name) else: data = conn.loadfile(r_name) - open(l_name, 'w').write(data) + open(l_name, 'wb').write(data) def copy_directory_local_to_remote(conn, localdir, remotedir): @@ -274,7 +274,7 @@ def copy_directory_local_to_remote(conn, localdir, remotedir): r_dirs.append(r_name) conn.mkdir(r_name) else: - data = open(l_name, 'r').read() + data = open(l_name, 'rb').read() conn.savefile(r_name, data) @@ -835,7 +835,7 @@ def run(self, gpo, H=None, tmpdir=None, sambaopts=None, credopts=None, versionop raise CommandError("GPO '%s' does not exist" % gpo) # verify UNC path - unc = msg['gPCFileSysPath'][0] + unc = msg['gPCFileSysPath'][0].decode('utf8') try: [dom_name, service, sharepath] = parse_unc(unc) except ValueError: @@ -1056,7 +1056,7 @@ def run(self, gpo, H=None, sambaopts=None, credopts=None, # Check if valid GPO try: msg = get_gpo_info(self.samdb, gpo=gpo)[0] - unc_path = msg['gPCFileSysPath'][0] + unc_path = msg['gPCFileSysPath'][0].decode('utf8') except Exception: raise CommandError("GPO '%s' does not exist" % gpo) @@ -1133,7 +1133,7 @@ def run(self, H=None, sambaopts=None, credopts=None, versionopts=None): for m in msg: # verify UNC path - unc = m['gPCFileSysPath'][0] + unc = m['gPCFileSysPath'][0].decode('utf8') try: [dom_name, service, sharepath] = parse_unc(unc) except ValueError: From a5b11d57b2785a0cfa196999b086837a0e3ab499 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 14:01:44 +0100 Subject: [PATCH 39/62] s4/selftest: enable samba.tests.samba_tool.gpo for python3 --- source4/selftest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 577e325b635..291e4c9f56b 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -599,8 +599,8 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex # We run this test against both AD DC implemetnations because it is # the only test we have of GPO get/set behaviour, and this involves # the file server as well as the LDAP server. -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.gpo") -planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.gpo") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.gpo", py3_compatible=True) +planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.gpo",py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.processes") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user") From d6b7202bbfb70a775b6d70f5a28c05e3623b8473 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 14:03:13 +0100 Subject: [PATCH 40/62] s4/selftest: enable samba.tests.samba_tool.gpo for python3 --- source4/selftest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 291e4c9f56b..28835c29c74 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -599,8 +599,8 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex # We run this test against both AD DC implemetnations because it is # the only test we have of GPO get/set behaviour, and this involves # the file server as well as the LDAP server. -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.gpo", py3_compatible=True) -planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.gpo",py3_compatible=True) +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.gpo", py3_compatible=True) +planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.gpo", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.processes") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user") From 36a844c985dfb94b035a19895b44e76ae9efa81b Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 14:07:05 +0100 Subject: [PATCH 41/62] s4/selftest: enable samba.tests.samba_tool.processes for python3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 28835c29c74..12a7d2a940c 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -602,7 +602,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.gpo", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.gpo", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.processes") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.processes", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user_wdigest") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user") From 57fd27330bf929b960caf8c76708195181ea05f8 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 15:16:11 +0100 Subject: [PATCH 42/62] python/samba/tests: use bytes for md5 --- python/samba/tests/password_hash.py | 4 ++++ python/samba/tests/samba_tool/user_wdigest.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/python/samba/tests/password_hash.py b/python/samba/tests/password_hash.py index a3a74aa25df..51703ef2e0f 100644 --- a/python/samba/tests/password_hash.py +++ b/python/samba/tests/password_hash.py @@ -34,6 +34,7 @@ import binascii from hashlib import md5 import crypt +from samba.compat import text_type USER_NAME = "PasswordHashTestUser" @@ -60,6 +61,9 @@ def get_package(sc, name): def calc_digest(user, realm, password): data = "%s:%s:%s" % (user, realm, password) + if isinstance(data, text_type): + data = data.encode('utf8') + return md5(data).hexdigest() diff --git a/python/samba/tests/samba_tool/user_wdigest.py b/python/samba/tests/samba_tool/user_wdigest.py index 35283ebfcb3..7156c2baf7e 100644 --- a/python/samba/tests/samba_tool/user_wdigest.py +++ b/python/samba/tests/samba_tool/user_wdigest.py @@ -33,6 +33,7 @@ from hashlib import md5 import random import string +from samba.compat import text_type USER_NAME = "WdigestTestUser" # Create a random 32 character password, containing only letters and @@ -45,6 +46,8 @@ # def calc_digest(user, realm, password): data = "%s:%s:%s" % (user, realm, password) + if isinstance(data, text_type): + data = data.encode('utf8') return "%s:%s:%s" % (user, realm, md5(data).hexdigest()) From b3b90f75b4e89628aa815f7067fe017a502853ec Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 18:21:05 +0100 Subject: [PATCH 43/62] python/samba/tests: samba.tests.samba_tool.user_wdigest py2/py3 compat --- python/samba/netcmd/user.py | 5 ++++- python/samba/tests/samba_tool/user_wdigest.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/samba/netcmd/user.py b/python/samba/netcmd/user.py index dfe167d8c7d..dbf6256fbb8 100644 --- a/python/samba/netcmd/user.py +++ b/python/samba/netcmd/user.py @@ -55,6 +55,7 @@ Option, ) +from samba.compat import string_types try: import io @@ -975,7 +976,7 @@ def get_account_attributes(self, samdb, username, basedn, filter, scope, unicodePwd = obj["unicodePwd"][0] if add_unicodePwd: del obj["unicodePwd"] - account_name = obj["sAMAccountName"][0] + account_name = obj["sAMAccountName"][0].decode('utf8') if add_sAMAcountName: del obj["sAMAccountName"] if "userPrincipalName" in obj: @@ -1157,6 +1158,8 @@ def get_wDigest(i, primary_wdigest, account_name, account_upn, primary_wdigest) try: digest = binascii.hexlify(bytearray(digests.hashes[i-1].hash)) + if not isinstance(digest, string_types): + digest = digest.decode('utf8') return "%s:%s:%s" % (user, realm, digest) except IndexError: return None diff --git a/python/samba/tests/samba_tool/user_wdigest.py b/python/samba/tests/samba_tool/user_wdigest.py index 7156c2baf7e..6937ecc3925 100644 --- a/python/samba/tests/samba_tool/user_wdigest.py +++ b/python/samba/tests/samba_tool/user_wdigest.py @@ -73,7 +73,7 @@ def setUp(self): base=self.samdb.get_config_basedn(), expression="ncName=%s" % self.samdb.get_default_basedn(), attrs=["nETBIOSName"]) - self.netbios_domain = res[0]["nETBIOSName"][0] + self.netbios_domain = res[0]["nETBIOSName"][0].decode('utf8') self.runsubcmd("user", "create", USER_NAME, From 509f45bfd145571106786a1c428a9ffbf0d963a9 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Mon, 23 Apr 2018 18:22:08 +0100 Subject: [PATCH 44/62] s4/selftest: enable samba.tests.samba_tool.user_wdigest for python3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 12a7d2a940c..5dad9a0c7fd 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -604,7 +604,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.processes", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user") -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user_wdigest") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user_wdigest", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user_virtualCryptSHA") planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script") From acb7610833d61cdd538981bb7cd1a4e73ad757cb Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 16:28:41 +0100 Subject: [PATCH 45/62] python/samba: Add binary_type for p2/p3 testing For helping test for binary types, binary_type evaluates to 'str' in py2, and 'bytes' in py3. --- python/samba/compat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/samba/compat.py b/python/samba/compat.py index 5662095fed5..042fc86a440 100644 --- a/python/samba/compat.py +++ b/python/samba/compat.py @@ -30,6 +30,7 @@ integer_types = int, string_types = str text_type = str + binary_type = bytes # alias import io @@ -43,6 +44,7 @@ integer_types = (int, long) string_types = basestring text_type = unicode + binary_type = str # alias import StringIO From 1092b114a5a507997dab3216ceeb8bea8caa6dff Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 16:37:18 +0100 Subject: [PATCH 46/62] python/samba/netcmd: Fix exception tuple assignment assigning a tuple to exception doesn't work in py3, for code to be py2/p3 compatable you assign to exception.args instead --- python/samba/netcmd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/netcmd/__init__.py b/python/samba/netcmd/__init__.py index 77976780150..9df096d5bd3 100644 --- a/python/samba/netcmd/__init__.py +++ b/python/samba/netcmd/__init__.py @@ -103,7 +103,7 @@ def show_command_error(self, e): force_traceback = True if isinstance(inner_exception, LdbError): - (ldb_ecode, ldb_emsg) = inner_exception + (ldb_ecode, ldb_emsg) = inner_exception.args self.errf.write("ERROR(ldb): %s - %s\n" % (message, ldb_emsg)) elif isinstance(inner_exception, AssertionError): self.errf.write("ERROR(assert): %s\n" % message) From 87bce69bb74c51b9368816cef36f293582ab84ae Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 16:40:00 +0100 Subject: [PATCH 47/62] python/samba/netcmd: Fix possible unicode error #TODO Check this This patch sortof replaces the change made with b86c29b1e6fb for this file. This change seems strange that a string would be encoded (e.g. converted to bytes) to be passed to an api that takes a string paramater. That is change_password takes a 'string' as a password. Alternatively change_password would take bytes and the previous change would be correct --- python/samba/netcmd/user.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/samba/netcmd/user.py b/python/samba/netcmd/user.py index dbf6256fbb8..d0ceb31f3c8 100644 --- a/python/samba/netcmd/user.py +++ b/python/samba/netcmd/user.py @@ -56,6 +56,7 @@ ) from samba.compat import string_types +from samba.compat import binary_type try: import io @@ -714,7 +715,9 @@ def run(self, credopts=None, sambaopts=None, versionopts=None, self.outf.write("Sorry, passwords do not match.\n") try: - net.change_password(password.encode('utf-8')) + if isinstance(password, binary_type): + password = password.decode('utf8') + net.change_password(password) except Exception as msg: # FIXME: catch more specific exception raise CommandError("Failed to change password : %s" % msg) From 9a2e18db1a112bf265f219ae90f0063296762e6b Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 16:55:49 +0100 Subject: [PATCH 48/62] s4/selftest: enable samba.tests.samba_tool.user_check_password_script py3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 5dad9a0c7fd..1366abbd331 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -607,7 +607,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.user_wdigest", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user_virtualCryptSHA") -planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script") +planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer") From d477878ee006f46cfd3100b6bbb166e166462ad2 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 19:07:55 +0100 Subject: [PATCH 49/62] python/samba/netcmd: fix samba.test.group for py3 --- python/samba/netcmd/group.py | 4 ++-- python/samba/tests/samba_tool/group.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/samba/netcmd/group.py b/python/samba/netcmd/group.py index a4969cc6ba9..8213a2fe1d8 100644 --- a/python/samba/netcmd/group.py +++ b/python/samba/netcmd/group.py @@ -356,7 +356,7 @@ def run(self, sambaopts=None, credopts=None, versionopts=None, H=None, self.outf.write("\n") else: for msg in res: - self.outf.write("%s\n" % msg.get("samaccountname", idx=0)) + self.outf.write("%s\n" % msg.get("samaccountname", idx=0).decode('utf8')) class cmd_group_list_members(Command): """List all members of an AD group. @@ -416,7 +416,7 @@ def run(self, groupname, credopts=None, sambaopts=None, versionopts=None, H=None member_name = msg.get("samAccountName", idx=0) if member_name is None: member_name = msg.get("cn", idx=0) - self.outf.write("%s\n" % member_name) + self.outf.write("%s\n" % member_name.decode('utf8')) except Exception as e: raise CommandError('Failed to list members of "%s" group ' % groupname, e) diff --git a/python/samba/tests/samba_tool/group.py b/python/samba/tests/samba_tool/group.py index 914b8175d15..1ade63e4eb4 100644 --- a/python/samba/tests/samba_tool/group.py +++ b/python/samba/tests/samba_tool/group.py @@ -114,7 +114,7 @@ def test_list(self): self.assertTrue(len(grouplist) > 0, "no groups found in samdb") for groupobj in grouplist: - name = groupobj.get("samaccountname", idx=0) + name = groupobj.get("samaccountname", idx=0).decode('utf8') found = self.assertMatch(out, name, "group '%s' not found" % name) @@ -135,7 +135,7 @@ def test_listmembers(self): self.assertTrue(len(grouplist) > 0, "no groups found in samdb") for groupobj in grouplist: - name = groupobj.get("samAccountName", idx=0) + name = groupobj.get("samAccountName", idx=0).decode('utf8') found = self.assertMatch(out, name, "group '%s' not found" % name) def test_move(self): From c01ab6cb58f431b4fce282628a254d142c7b9ed1 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Tue, 24 Apr 2018 19:08:27 +0100 Subject: [PATCH 50/62] s4/selftest: enable samba.tests.group for py3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 1366abbd331..bd82d457147 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -608,7 +608,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user_virtualCryptSHA") planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou") planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl") From 8e7cd167c6ceadbd20728d3d70ee74830513f064 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Wed, 25 Apr 2018 11:06:09 +0100 Subject: [PATCH 51/62] python/samba: Py2/Py3 compat change '/' to '//' to ensure int result --- python/samba/graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/graph.py b/python/samba/graph.py index b8c5088bec3..98cd2a6e861 100644 --- a/python/samba/graph.py +++ b/python/samba/graph.py @@ -39,8 +39,8 @@ def reformat_graph_label(s): if '-' in p[2:20]: q, p = p.split('-', 1) else: - n = len(p) / 12 - b = len(p) / n + n = len(p) // 12 + b = len(p) // n q, p = p[:b], p[b:] pieces.append(q + '-') if p: From da90908b52db14865fc03ac3d1271fb1ddf8ae74 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 26 Apr 2018 15:47:19 +0100 Subject: [PATCH 52/62] s4/selftest: enable samba.tests.samba_tool.ou for py3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index bd82d457147..c1ee662e5db 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -609,7 +609,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.user_virtualCryptSHA") planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer") planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl") planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check") From 1750f7bcb2a62b6d364796d0d5ef59ab48356635 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 26 Apr 2018 16:02:10 +0100 Subject: [PATCH 53/62] python/samba: make Exception statment py3 compatible e.g. - except Exception, e: + except Exception as e: --- python/samba/netcmd/computer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/samba/netcmd/computer.py b/python/samba/netcmd/computer.py index 3b130b1fa25..9ca8904a70e 100644 --- a/python/samba/netcmd/computer.py +++ b/python/samba/netcmd/computer.py @@ -298,7 +298,7 @@ def run(self, computername, credopts=None, sambaopts=None, versionopts=None, samdb, hostname, dns_conn, change_owner_sd, samdb.host_dns_name(), ip_address_list, self.get_logger()) - except Exception, e: + except Exception as e: raise CommandError("Failed to create computer '%s': " % computername, e) @@ -394,7 +394,7 @@ def run(self, computername, credopts=None, sambaopts=None, remove_dns_references( samdb, self.get_logger(), computer_dns_host_name, ignore_no_name=True) - except Exception, e: + except Exception as e: raise CommandError('Failed to remove computer "%s"' % samaccountname, e) self.outf.write("Deleted computer %s\n" % computername) @@ -569,7 +569,7 @@ def run(self, computername, new_ou_dn, credopts=None, sambaopts=None, new_computer_dn.add_base(full_new_ou_dn) try: samdb.rename(computer_dn, new_computer_dn) - except Exception, e: + except Exception as e: raise CommandError('Failed to move computer "%s"' % computername, e) self.outf.write('Moved computer "%s" to "%s"\n' % (computername, new_ou_dn)) From 23f8e44e12962a4f7246fbd93a8f49a650838f86 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 26 Apr 2018 18:22:21 +0100 Subject: [PATCH 54/62] python/samba/netcmd: changes for samab.tests.samba_tool.computer --- python/samba/netcmd/computer.py | 4 ++-- python/samba/remove_dc.py | 2 +- python/samba/samdb.py | 2 +- python/samba/tests/samba_tool/computer.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/samba/netcmd/computer.py b/python/samba/netcmd/computer.py index 9ca8904a70e..e646cf4911b 100644 --- a/python/samba/netcmd/computer.py +++ b/python/samba/netcmd/computer.py @@ -281,7 +281,7 @@ def run(self, computername, credopts=None, sambaopts=None, versionopts=None, expression=filters, attrs=['primaryGroupID', 'objectSid']) - group = recs[0]['primaryGroupID'][0] + group = recs[0]['primaryGroupID'][0].decode('utf8') owner = ndr_unpack(security.dom_sid, recs[0]["objectSid"][0]) dns_conn = dnsserver.dnsserver( @@ -376,7 +376,7 @@ def run(self, computername, credopts=None, sambaopts=None, computer_dn = res[0].dn computer_ac = int(res[0]["userAccountControl"][0]) if "dNSHostName" in res[0]: - computer_dns_host_name = res[0]["dNSHostName"][0] + computer_dns_host_name = res[0]["dNSHostName"][0].decode('utf8') else: computer_dns_host_name = None except IndexError: diff --git a/python/samba/remove_dc.py b/python/samba/remove_dc.py index 6b86a554efa..1a442cc0d61 100644 --- a/python/samba/remove_dc.py +++ b/python/samba/remove_dc.py @@ -135,7 +135,7 @@ def dns_name_from_dn(dn): # By using a set here, duplicates via (eg) example.com/Configuration # do not matter, they become just example.com a_names_to_remove_from \ - = set(dns_name_from_dn(dn) for dn in ncs) + = set(dns_name_from_dn(dn.decode('utf8')) for dn in ncs) def a_rec_to_remove(dnsRecord): if dnsRecord.wType == DNS_TYPE_A or dnsRecord.wType == DNS_TYPE_AAAA: diff --git a/python/samba/samdb.py b/python/samba/samdb.py index a1a1f45b003..9f5a1631890 100644 --- a/python/samba/samdb.py +++ b/python/samba/samdb.py @@ -733,7 +733,7 @@ def server_site_name(self): def host_dns_name(self): """return the DNS name of this host""" res = self.search(base='', scope=ldb.SCOPE_BASE, attrs=['dNSHostName']) - return res[0]['dNSHostName'][0] + return res[0]['dNSHostName'][0].decode('utf8') def domain_dns_name(self): """return the DNS name of the domain root""" diff --git a/python/samba/tests/samba_tool/computer.py b/python/samba/tests/samba_tool/computer.py index 4036d973c12..52583601613 100644 --- a/python/samba/tests/samba_tool/computer.py +++ b/python/samba/tests/samba_tool/computer.py @@ -186,7 +186,7 @@ def test_list(self): self.assertTrue(len(computerlist) > 0, "no computers found in samdb") for computerobj in computerlist: - name = computerobj.get("samaccountname", idx=0) + name = computerobj.get("samaccountname", idx=0).decode('utf8') found = self.assertMatch(out, name, "computer '%s' not found" % name) @@ -265,7 +265,7 @@ def _create_computer(self, computer): args += ' --service-principal-name={}'.format(service_principal_name) args = args.split() - + print ('#### args %s %s' % (type(args), args)) return self.runsubcmd('computer', 'create', *args) def _create_ou(self, ou): @@ -325,5 +325,5 @@ def _find_service_principal_name(self, name, expected_service_principal_names): names = set() for computer in computer_list: for name in computer.get('servicePrincipalName', []): - names.add(name) + names.add(name.decode('utf8')) return names == set(expected_service_principal_names) From 48185260bc1858ba821fa5b39aa0c101ef1b937a Mon Sep 17 00:00:00 2001 From: Noel Power Date: Thu, 26 Apr 2018 18:23:50 +0100 Subject: [PATCH 55/62] s4/selftest/tests: Enabled samba.tests.samba_tool.computer --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index c1ee662e5db..5b4d737389d 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -610,7 +610,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.user_check_password_script", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou", py3_compatible=True) -planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer") +planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl") planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check") planpythontestsuite("none", "samba.tests.samba_tool.help") From ba24d3526bb8747c2e0900703766216668c5fdab Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 08:14:27 +0100 Subject: [PATCH 56/62] squash with python/samba: Py2/Py3 compat change '/' to '//' to ensure int result --- python/samba/netcmd/domain.py | 2 +- python/samba/upgradehelpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/netcmd/domain.py b/python/samba/netcmd/domain.py index d2dd06a3d48..a6de29294cb 100644 --- a/python/samba/netcmd/domain.py +++ b/python/samba/netcmd/domain.py @@ -2331,7 +2331,7 @@ def get_password(name): # 512 bytes and a 2 bytes confounder is required. # def random_trust_secret(length): - pw = samba.generate_random_machine_password(length/2, length/2) + pw = samba.generate_random_machine_password(length//2, length//2) return string_to_byte_array(pw.encode('utf-16-le')) if local_trust_info.trust_direction & lsa.LSA_TRUST_DIRECTION_INBOUND: diff --git a/python/samba/upgradehelpers.py b/python/samba/upgradehelpers.py index e219602e0c7..7afdbc64ffc 100644 --- a/python/samba/upgradehelpers.py +++ b/python/samba/upgradehelpers.py @@ -725,7 +725,7 @@ def findprovisionrange(samdb, basedn): for o in obj.array: # like a timestamp but with the resolution of 1 minute - minutestamp =_glue.nttime2unix(o.originating_change_time)/60 + minutestamp =_glue.nttime2unix(o.originating_change_time) // 60 hash_ts = hash_id.get(str(o.originating_invocation_id)) if hash_ts is None: From 996dbc09422d39a849ef7436373da00ea71b12a7 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 13:02:50 +0100 Subject: [PATCH 57/62] python/samba/provision: remove use of str() func for binary data Python 2 code works with str(policy["nTSecurityDescriptor"]) however this cannot work with Python 3. One could argue even the str method doesn't make sense at all (returning a string) for data. Signed-off-by: Noel Power --- python/samba/provision/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py index fc308fcad26..a4024114a13 100644 --- a/python/samba/provision/__init__.py +++ b/python/samba/provision/__init__.py @@ -27,6 +27,7 @@ __docformat__ = "restructuredText" from samba.compat import string_types from samba.compat import urllib_quote +from samba.compat import text_type from base64 import b64encode import errno import os @@ -1607,7 +1608,7 @@ def set_gpos_acl(sysvol, dnsdomain, domainsid, domaindn, samdb, lp, use_ntvfs, p for policy in res: acl = ndr_unpack(security.descriptor, - str(policy["nTSecurityDescriptor"])).as_sddl() + policy["nTSecurityDescriptor"][0]).as_sddl() policy_path = getpolicypath(sysvol, dnsdomain, str(policy["cn"])) set_dir_acl(policy_path, dsacl2fsacl(acl, domainsid), lp, str(domainsid), use_ntvfs, @@ -1768,7 +1769,7 @@ def check_gpos_acl(sysvol, dnsdomain, domainsid, domaindn, samdb, lp, for policy in res: acl = ndr_unpack(security.descriptor, - str(policy["nTSecurityDescriptor"])).as_sddl() + policy["nTSecurityDescriptor"][0]).as_sddl() policy_path = getpolicypath(sysvol, dnsdomain, str(policy["cn"])) check_dir_acl(policy_path, dsacl2fsacl(acl, domainsid), lp, domainsid, direct_db_access) From 30e9fda0e38069f6a8c359c4cddd2e9ec97ed825 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 13:05:15 +0100 Subject: [PATCH 58/62] s4/selftest: enable samba.tests.samba_tool.ntacl for Py3 Signed-off-by: Noel Power --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 5b4d737389d..51deaac498c 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -611,7 +611,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.group", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer", py3_compatible=True) -planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl") +planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl", py3_compatible=True) planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check") planpythontestsuite("none", "samba.tests.samba_tool.help") From 5234cbb892656947e59a9bb2d995dc42fa624546 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 15:11:54 +0100 Subject: [PATCH 59/62] python/samba/netcmd: Fix password usage for py2/py3 compatability getpass returns str (e.g. bytes) in python2 and str (unicode) in py3. Adapt code to so we don't do illegal things (like try and decode) a string in python3 Signed-off-by: Noel Power --- python/samba/netcmd/domain.py | 7 +++++-- python/samba/provision/__init__.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/samba/netcmd/domain.py b/python/samba/netcmd/domain.py index a6de29294cb..be47e511e73 100644 --- a/python/samba/netcmd/domain.py +++ b/python/samba/netcmd/domain.py @@ -98,6 +98,8 @@ FILL_DRS ) +from samba.compat import binary_type + string_version_to_constant = { "2008_R2" : DS_DOMAIN_FUNCTION_2008_R2, "2012": DS_DOMAIN_FUNCTION_2012, @@ -539,8 +541,9 @@ def _get_nameserver_ip(self): def _adminpass_issue(self, adminpass): """Returns error string for a bad administrator password, or None if acceptable""" - - if len(adminpass.decode('utf-8')) < DEFAULT_MIN_PWD_LENGTH: + if isinstance(adminpass, binary_type): + adminpass = adminpass.decode('utf8') + if len(adminpass) < DEFAULT_MIN_PWD_LENGTH: return "Administrator password does not meet the default minimum" \ " password length requirement (%d characters)" \ % DEFAULT_MIN_PWD_LENGTH diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py index a4024114a13..9c6a0f14d5a 100644 --- a/python/samba/provision/__init__.py +++ b/python/samba/provision/__init__.py @@ -28,6 +28,7 @@ from samba.compat import string_types from samba.compat import urllib_quote from samba.compat import text_type +from samba.compat import binary_type from base64 import b64encode import errno import os @@ -2277,7 +2278,8 @@ def provision(logger, session_info, smbconf=None, adminpass = samba.generate_random_password(12, 32) adminpass_generated = True else: - adminpass = unicode(adminpass, 'utf-8') + if isinstance(adminpass, binary_type): + adminpass = unicode(adminpass, 'utf-8') adminpass_generated = False if samdb_fill == FILL_FULL: From ab71e5a3833bcafcaa2cc474807db0b4982db1ff Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 15:13:49 +0100 Subject: [PATCH 60/62] s4/selftest: Enable samba.tests.samba_tool.provision_password_check for py3 Signed-off-by: Noel Power --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 51deaac498c..69b28138a55 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -612,7 +612,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.ou", py3_compatible=True) planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl", py3_compatible=True) -planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check") +planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check", py3_compatible=True) planpythontestsuite("none", "samba.tests.samba_tool.help") # Run these against chgdcpass to share the runtime load From 7614efa88734bdc49358217ddb1855f5020314f1 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 15:45:49 +0100 Subject: [PATCH 61/62] python/samba/tests: Py2/Py2 ensure bytes output is converted to text type Tests in python3 are intermingling bytes and strings types. Adjust code to ensure bytes are converted to str as appropriate --- python/samba/tests/samba_tool/help.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/samba/tests/samba_tool/help.py b/python/samba/tests/samba_tool/help.py index 62e0b3eaa23..853602c053c 100644 --- a/python/samba/tests/samba_tool/help.py +++ b/python/samba/tests/samba_tool/help.py @@ -20,7 +20,7 @@ import re from samba.tests.samba_tool.base import SambaToolCmdTest from samba.tests import BlackboxProcessError - +from samba.compat import binary_type class HelpTestCase(SambaToolCmdTest): """Tests for samba-tool help and --help @@ -47,7 +47,8 @@ def test_help_tree(self): except BlackboxProcessError as e: output = e.stdout failed_commands.append(c) - + if isinstance(output, binary_type): + output = output.decode('utf8') tail = output.partition('Available subcommands:')[2] subcommands = re.findall(r'^\s*([\w-]+)\s+-', tail, re.MULTILINE) @@ -61,7 +62,8 @@ def test_help_tree(self): except BlackboxProcessError as e: output2 = e.stdout failed_commands.append(c) - + if isinstance(output2, binary_type): + output2 = output2.decode('utf8') self.assertEqual(output, output2) if not new_commands: From b7c2ae59309e56cbbeba29b5435323d24245c169 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 27 Apr 2018 15:49:40 +0100 Subject: [PATCH 62/62] s4/selftest: enable samba.tests.samba_tool.help for py3 --- source4/selftest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 69b28138a55..6f1c262611f 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -613,7 +613,7 @@ def planoldpythontestsuite(env, module, name=None, extra_path=[], environ={}, ex planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.samba_tool.computer", py3_compatible=True) planpythontestsuite("ad_dc:local", "samba.tests.samba_tool.ntacl", py3_compatible=True) planpythontestsuite("none", "samba.tests.samba_tool.provision_password_check", py3_compatible=True) -planpythontestsuite("none", "samba.tests.samba_tool.help") +planpythontestsuite("none", "samba.tests.samba_tool.help", py3_compatible=True) # Run these against chgdcpass to share the runtime load planpythontestsuite("chgdcpass:local", "samba.tests.samba_tool.sites")