[PATCH] LMDB key value backend for ldb_tdb (to be renamed ldb_key_val)

Andrew Bartlett abartlet at samba.org
Wed Apr 4 05:54:22 UTC 2018


On Wed, 2018-04-04 at 06:57 +1200, Andrew Bartlett wrote:
> 
> However I will work today to see if anything can be pulled forward
> without upending the entire patch set.  It may be possible to bring
> some tests forward without the build hook for the lmdb mode, for
> example. 

I've spend my entire work day today re-ordering the patch set.  I hope
this is now satisfactory, as I can't put much more time into this.

Attached are two patches, one on top of the other patches sent today,
and one with all the LDB patches I'm trying to get merged.

Each patch compiles on its own and passes the ldb tests on its own.

If you could let me know that you are happy that I've addressed your
concerns as far as is practical that would be most appreciated. 
(Review comments also most welcome). 

The full CI results will be here:
https://gitlab.com/catalyst-samba/samba/pipelines/19933665

I've got one last patch ordering issue to work out with the
upgradeprovision.alpha13 test, but otherwise I do think this is good to
go.

Thanks,

Andrew Bartlett
-- 
Andrew Bartlett
https://samba.org/~abartlet/
Authentication Developer, Samba Team         https://samba.org
Samba Development and Support, Catalyst IT   
https://catalyst.net.nz/services/samba



-------------- next part --------------
From 216b54de513974d6496eb2abed56034913d653cb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 26 Mar 2018 16:01:13 +1300
Subject: [PATCH 01/54] ldb_tdb: Ensure we can not commit an index that is
 corrupt due to partial re-index

The re-index traverse can abort part-way though and we need to ensure
that the transaction is never committed as that will leave an un-useable db.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13335

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 30 ++++++++++++++++++++++++++++++
 lib/ldb/ldb_tdb/ldb_tdb.h |  2 ++
 2 files changed, 32 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 9c8c77155c4..20796f2162d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -410,6 +410,10 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
 		ret = ltdb_cache_reload(module);
 	}
 
+	if (ret != LDB_SUCCESS) {
+		ltdb->reindex_failed = true;
+	}
+
 	return ret;
 }
 
@@ -1443,9 +1447,17 @@ static int ltdb_start_trans(struct ldb_module *module)
 
 	ltdb_index_transaction_start(module);
 
+	ltdb->reindex_failed = false;
+
 	return LDB_SUCCESS;
 }
 
+/*
+ * Forward declaration to allow prepare_commit to in fact abort the
+ * transaction
+ */
+static int ltdb_del_trans(struct ldb_module *module);
+
 static int ltdb_prepare_commit(struct ldb_module *module)
 {
 	int ret;
@@ -1456,6 +1468,24 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 		return LDB_SUCCESS;
 	}
 
+	/*
+	 * Check if the last re-index failed.
+	 *
+	 * This can happen if for example a duplicate value was marked
+	 * unique.  We must not write a partial re-index into the DB.
+	 */
+	if (ltdb->reindex_failed) {
+		/*
+		 * We must instead abort the transaction so we get the
+		 * old values and old index back
+		 */
+		ltdb_del_trans(module);
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				  "Failure during re-index, so "
+				  "transaction must be aborted.");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
 	ret = ltdb_index_transaction_commit(module);
 	if (ret != LDB_SUCCESS) {
 		ltdb->kv_ops->abort_write(ltdb);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 2235bd47c98..28dd20915c7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -65,6 +65,8 @@ struct ltdb_private {
 
 	bool read_only;
 
+	bool reindex_failed;
+
 	const struct ldb_schema_syntax *GUID_index_syntax;
 
 	/*
-- 
2.11.0


From 95c83217f0d5b8a886986cd2ab06493df70a517b Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 02/54] lib ldb tests: Prepare to run api and index test on tdb
 and lmdb

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13335

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py   | 145 +++++++++++++++++++++++++-----------------
 lib/ldb/tests/python/index.py |  29 ++++++++-
 2 files changed, 114 insertions(+), 60 deletions(-)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 85fe1bc360f..51c9c592f78 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -12,6 +12,9 @@ import shutil
 
 PY3 = sys.version_info > (3, 0)
 
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
 
 def tempdir():
     import tempfile
@@ -44,13 +47,36 @@ class NoContextTests(TestCase):
         encoded2 = ldb.binary_encode('test\\x')
         self.assertEqual(encoded2, encoded)
 
-class SimpleLdb(TestCase):
+
+class LdbBaseTest(TestCase):
+    def setUp(self):
+        super(LdbBaseTest, self).setUp()
+        try:
+            if self.prefix is None:
+                self.prefix = TDB_PREFIX
+        except AttributeError:
+            self.prefix = TDB_PREFIX
+
+    def tearDown(self):
+        super(LdbBaseTest, self).tearDown()
+
+    def url(self):
+        return self.prefix + self.filename
+
+    def flags(self):
+        if self.prefix == MDB_PREFIX:
+            return ldb.FLG_NOSYNC
+        else:
+            return 0
+
+
+class SimpleLdb(LdbBaseTest):
 
     def setUp(self):
         super(SimpleLdb, self).setUp()
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
-        self.ldb = ldb.Ldb(self.filename)
+        self.ldb = ldb.Ldb(self.url(), flags=self.flags())
 
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -58,16 +84,15 @@ class SimpleLdb(TestCase):
         # Ensure the LDB is closed now, so we close the FD
         del(self.ldb)
 
-
     def test_connect(self):
-        ldb.Ldb(self.filename)
+        ldb.Ldb(self.url(), flags=self.flags())
 
     def test_connect_none(self):
         ldb.Ldb()
 
     def test_connect_later(self):
         x = ldb.Ldb()
-        x.connect(self.filename)
+        x.connect(self.url(), flags=self.flags())
 
     def test_repr(self):
         x = ldb.Ldb()
@@ -82,7 +107,7 @@ class SimpleLdb(TestCase):
         self.assertEqual([], x.modules())
 
     def test_modules_tdb(self):
-        x = ldb.Ldb(self.filename)
+        x = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual("[<ldb module 'tdb'>]", repr(x.modules()))
 
     def test_firstmodule_none(self):
@@ -90,53 +115,53 @@ class SimpleLdb(TestCase):
         self.assertEqual(x.firstmodule, None)
 
     def test_firstmodule_tdb(self):
-        x = ldb.Ldb(self.filename)
+        x = ldb.Ldb(self.url(), flags=self.flags())
         mod = x.firstmodule
         self.assertEqual(repr(mod), "<ldb module 'tdb'>")
 
     def test_search(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search()), 0)
 
     def test_search_controls(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search(controls=["paged_results:0:5"])), 0)
 
     def test_search_attrs(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search(ldb.Dn(l, ""), ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
 
     def test_search_string_dn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search("", ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
 
     def test_search_attr_string(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertRaises(TypeError, l.search, attrs="dc")
         self.assertRaises(TypeError, l.search, attrs=b"dc")
 
     def test_opaque(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         l.set_opaque("my_opaque", l)
         self.assertTrue(l.get_opaque("my_opaque") is not None)
         self.assertEqual(None, l.get_opaque("unknown"))
 
     def test_search_scope_base_empty_db(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
                           ldb.SCOPE_BASE)), 0)
 
     def test_search_scope_onelevel_empty_db(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
                           ldb.SCOPE_ONELEVEL)), 0)
 
     def test_delete(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertRaises(ldb.LdbError, lambda: l.delete(ldb.Dn(l, "dc=foo2")))
 
     def test_delete_w_unhandled_ctrl(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo1")
         m["b"] = [b"a"]
@@ -145,10 +170,10 @@ class SimpleLdb(TestCase):
         l.delete(m.dn)
 
     def test_contains(self):
-        name = self.filename
-        l = ldb.Ldb(name)
+        name = self.url()
+        l = ldb.Ldb(name, flags=self.flags())
         self.assertFalse(ldb.Dn(l, "dc=foo3") in l)
-        l = ldb.Ldb(name)
+        l = ldb.Ldb(name, flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo3")
         m["b"] = ["a"]
@@ -160,23 +185,23 @@ class SimpleLdb(TestCase):
             l.delete(m.dn)
 
     def test_get_config_basedn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(None, l.get_config_basedn())
 
     def test_get_root_basedn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(None, l.get_root_basedn())
 
     def test_get_schema_basedn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(None, l.get_schema_basedn())
 
     def test_get_default_basedn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(None, l.get_default_basedn())
 
     def test_add(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = b"bla"
@@ -188,7 +213,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo4"))
 
     def test_search_iterator(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         s = l.search_iterator()
         s.abandon()
         try:
@@ -288,7 +313,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo5"))
 
     def test_add_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = "bla"
@@ -300,7 +325,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo4"))
 
     def test_add_w_unhandled_ctrl(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = b"bla"
@@ -308,7 +333,7 @@ class SimpleLdb(TestCase):
         self.assertRaises(ldb.LdbError, lambda: l.add(m,["search_options:1:2"]))
 
     def test_add_dict(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
              "bla": b"bla"}
         self.assertEqual(len(l.search()), 0)
@@ -319,7 +344,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo5"))
 
     def test_add_dict_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
              "bla": "bla"}
         self.assertEqual(len(l.search()), 0)
@@ -330,7 +355,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo5"))
 
     def test_add_dict_string_dn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": "dc=foo6", "bla": b"bla"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
@@ -340,7 +365,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo6"))
 
     def test_add_dict_bytes_dn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": b"dc=foo6", "bla": b"bla"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
@@ -350,7 +375,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=foo6"))
 
     def test_rename(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo7")
         m["bla"] = b"bla"
@@ -363,7 +388,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=bar"))
 
     def test_rename_string_dns(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo8")
         m["bla"] = b"bla"
@@ -377,7 +402,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=bar"))
 
     def test_empty_dn(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertEqual(0, len(l.search()))
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=empty")
@@ -394,7 +419,7 @@ class SimpleLdb(TestCase):
         self.assertEqual(0, len(rm[0]))
 
     def test_modify_delete(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m["bla"] = [b"1234"]
@@ -417,7 +442,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=modifydelete"))
 
     def test_modify_delete_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m.text["bla"] = ["1234"]
@@ -440,7 +465,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=modifydelete"))
 
     def test_modify_add(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
@@ -458,7 +483,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=add"))
 
     def test_modify_add_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
@@ -476,7 +501,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=add"))
 
     def test_modify_replace(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m["bla"] = [b"1234", b"456"]
@@ -496,7 +521,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=modify2"))
 
     def test_modify_replace_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m.text["bla"] = ["1234", "456"]
@@ -516,7 +541,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=modify2"))
 
     def test_modify_flags_change(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
@@ -542,7 +567,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=add"))
 
     def test_modify_flags_change_text(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
@@ -568,7 +593,7 @@ class SimpleLdb(TestCase):
             l.delete(ldb.Dn(l, "dc=add"))
 
     def test_transaction_commit(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo9"))
         m["foo"] = [b"bar"]
@@ -577,7 +602,7 @@ class SimpleLdb(TestCase):
         l.delete(m.dn)
 
     def test_transaction_cancel(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo10"))
         m["foo"] = [b"bar"]
@@ -588,12 +613,12 @@ class SimpleLdb(TestCase):
     def test_set_debug(self):
         def my_report_fn(level, text):
             pass
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         l.set_debug(my_report_fn)
 
     def test_zero_byte_string(self):
         """Testing we do not get trapped in the \0 byte in a property string."""
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         l.add({
             "dn" : b"dc=somedn",
             "objectclass" : b"user",
@@ -605,10 +630,10 @@ class SimpleLdb(TestCase):
         self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
 
     def test_no_crash_broken_expr(self):
-        l = ldb.Ldb(self.filename)
+        l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
 
-class SearchTests(TestCase):
+class SearchTests(LdbBaseTest):
     def tearDown(self):
         shutil.rmtree(self.testdir)
         super(SearchTests, self).tearDown()
@@ -621,7 +646,9 @@ class SearchTests(TestCase):
         super(SearchTests, self).setUp()
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "search_test.ldb")
-        self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+        self.l = ldb.Ldb(self.url(),
+                         flags=self.flags(),
+                         options=["modules:rdn_name"])
 
         self.l.add({"dn": "@ATTRIBUTES",
                     "DC": "CASE_INSENSITIVE"})
@@ -1030,7 +1057,6 @@ class SearchTests(TestCase):
         self.assertEqual(len(res11), 1)
 
 
-
 class IndexedSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1091,6 +1117,7 @@ class GUIDIndexedSearchTests(SearchTests):
         self.IDXGUID = True
         self.IDXONE = True
 
+
 class GUIDIndexedDNFilterSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1126,7 +1153,7 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
         self.IDXONE = True
 
 
-class AddModifyTests(TestCase):
+class AddModifyTests(LdbBaseTest):
     def tearDown(self):
         shutil.rmtree(self.testdir)
         super(AddModifyTests, self).tearDown()
@@ -1138,7 +1165,9 @@ class AddModifyTests(TestCase):
         super(AddModifyTests, self).setUp()
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "add_test.ldb")
-        self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+        self.l = ldb.Ldb(self.url(),
+                         flags=self.flags(),
+                         options=["modules:rdn_name"])
         self.l.add({"dn": "DC=SAMBA,DC=ORG",
                     "name": b"samba.org",
                     "objectUUID": b"0123456789abcdef"})
@@ -1266,6 +1295,7 @@ class AddModifyTests(TestCase):
                     "x": "z", "y": "a",
                     "objectUUID": b"0123456789abcde3"})
 
+
 class IndexedAddModifyTests(AddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1378,7 +1408,6 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
         super(TransIndexedAddModifyTests, self).tearDown()
 
 
-
 class DnTests(TestCase):
 
     def setUp(self):
@@ -1989,13 +2018,13 @@ class ModuleTests(TestCase):
         l = ldb.Ldb(self.filename)
         self.assertEqual(["init"], ops)
 
-class LdbResultTests(TestCase):
+class LdbResultTests(LdbBaseTest):
 
     def setUp(self):
         super(LdbResultTests, self).setUp()
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
-        self.l = ldb.Ldb(self.filename)
+        self.l = ldb.Ldb(self.url(), flags=self.flags())
         self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
         self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
         self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
@@ -2103,7 +2132,7 @@ class LdbResultTests(TestCase):
             del(self.l)
             gc.collect()
 
-            child_ldb = ldb.Ldb(self.filename)
+            child_ldb = ldb.Ldb(self.url(), flags=self.flags())
             # start a transaction
             child_ldb.transaction_start()
 
@@ -2174,7 +2203,7 @@ class LdbResultTests(TestCase):
             del(self.l)
             gc.collect()
 
-            child_ldb = ldb.Ldb(self.filename)
+            child_ldb = ldb.Ldb(self.url(), flags=self.flags())
             # start a transaction
             child_ldb.transaction_start()
 
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 239b2bff3fd..e9eaaf059e1 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -32,6 +32,9 @@ import shutil
 
 PY3 = sys.version_info > (3, 0)
 
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
 
 def tempdir():
     import tempfile
@@ -52,7 +55,29 @@ def contains(result, dn):
     return False
 
 
-class MaxIndexKeyLengthTests(TestCase):
+class LdbBaseTest(TestCase):
+    def setUp(self):
+        super(LdbBaseTest, self).setUp()
+        try:
+            if self.prefix is None:
+                self.prefix = TDB_PREFIX
+        except AttributeError:
+            self.prefix = TDB_PREFIX
+
+    def tearDown(self):
+        super(LdbBaseTest, self).tearDown()
+
+    def url(self):
+        return self.prefix + self.filename
+
+    def flags(self):
+        if self.prefix == MDB_PREFIX:
+            return ldb.FLG_NOSYNC
+        else:
+            return 0
+
+
+class MaxIndexKeyLengthTests(LdbBaseTest):
     def checkGuids(self, key, guids):
         #
         # This check relies on the current implementation where the indexes
@@ -89,7 +114,7 @@ class MaxIndexKeyLengthTests(TestCase):
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "key_len_test.ldb")
         # Note that the maximum key length is set to 50
-        self.l = ldb.Ldb(self.filename,
+        self.l = ldb.Ldb(self.url(),
                          options=[
                              "modules:rdn_name",
                              "max_key_len_for_self_test:50"])
-- 
2.11.0


From b4125238ac8bc2f83ef68abf20a8c979562ff29e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 26 Mar 2018 16:07:45 +1300
Subject: [PATCH 03/54] ldb: Add test to show a reindex failure must not leave
 the DB corrupt

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13335

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py | 160 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 160 insertions(+)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 51c9c592f78..12409a8c991 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -1408,6 +1408,166 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
         super(TransIndexedAddModifyTests, self).tearDown()
 
 
+class BadIndexTests(LdbBaseTest):
+    def setUp(self):
+        super(BadIndexTests, self).setUp()
+        self.testdir = tempdir()
+        self.filename = os.path.join(self.testdir, "test.ldb")
+        self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+        if hasattr(self, 'IDXGUID'):
+            self.ldb.add({"dn": "@INDEXLIST",
+                          "@IDXATTR": [b"x", b"y", b"ou"],
+                          "@IDXGUID": [b"objectUUID"],
+                          "@IDX_DN_GUID": [b"GUID"]})
+        else:
+            self.ldb.add({"dn": "@INDEXLIST",
+                          "@IDXATTR": [b"x", b"y", b"ou"]})
+
+        super(BadIndexTests, self).setUp()
+
+    def test_unique(self):
+        self.ldb.add({"dn": "x=x,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde1",
+                      "y": "1"})
+        self.ldb.add({"dn": "x=y,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde2",
+                      "y": "1"})
+        self.ldb.add({"dn": "x=z,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde3",
+                      "y": "1"})
+
+        res = self.ldb.search(expression="(y=1)",
+                              base="dc=samba,dc=org")
+        self.assertEquals(len(res), 3)
+
+        # Now set this to unique index, but forget to check the result
+        try:
+            self.ldb.add({"dn": "@ATTRIBUTES",
+                        "y": "UNIQUE_INDEX"})
+            self.fail()
+        except ldb.LdbError:
+            pass
+
+        # We must still have a working index
+        res = self.ldb.search(expression="(y=1)",
+                              base="dc=samba,dc=org")
+        self.assertEquals(len(res), 3)
+
+    def test_unique_transaction(self):
+        self.ldb.add({"dn": "x=x,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde1",
+                      "y": "1"})
+        self.ldb.add({"dn": "x=y,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde2",
+                      "y": "1"})
+        self.ldb.add({"dn": "x=z,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde3",
+                      "y": "1"})
+
+        res = self.ldb.search(expression="(y=1)",
+                              base="dc=samba,dc=org")
+        self.assertEquals(len(res), 3)
+
+        self.ldb.transaction_start()
+
+        # Now set this to unique index, but forget to check the result
+        try:
+            self.ldb.add({"dn": "@ATTRIBUTES",
+                        "y": "UNIQUE_INDEX"})
+        except ldb.LdbError:
+            pass
+
+        try:
+            self.ldb.transaction_commit()
+            self.fail()
+
+        except ldb.LdbError as err:
+            enum = err.args[0]
+            self.assertEqual(enum, ldb.ERR_OPERATIONS_ERROR)
+
+        # We must still have a working index
+        res = self.ldb.search(expression="(y=1)",
+                              base="dc=samba,dc=org")
+
+        self.assertEquals(len(res), 3)
+
+    def test_casefold(self):
+        self.ldb.add({"dn": "x=x,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde1",
+                      "y": "a"})
+        self.ldb.add({"dn": "x=y,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde2",
+                      "y": "A"})
+        self.ldb.add({"dn": "x=z,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde3",
+                      "y": ["a", "A"]})
+
+        res = self.ldb.search(expression="(y=a)",
+                              base="dc=samba,dc=org")
+        self.assertEquals(len(res), 2)
+
+        self.ldb.add({"dn": "@ATTRIBUTES",
+                      "y": "CASE_INSENSITIVE"})
+
+        # We must still have a working index
+        res = self.ldb.search(expression="(y=a)",
+                              base="dc=samba,dc=org")
+
+        if hasattr(self, 'IDXGUID'):
+            self.assertEquals(len(res), 3)
+        else:
+            # We should not return this entry twice, but sadly
+            # we have not yet fixed
+            # https://bugzilla.samba.org/show_bug.cgi?id=13361
+            self.assertEquals(len(res), 4)
+
+    def test_casefold_transaction(self):
+        self.ldb.add({"dn": "x=x,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde1",
+                      "y": "a"})
+        self.ldb.add({"dn": "x=y,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde2",
+                      "y": "A"})
+        self.ldb.add({"dn": "x=z,dc=samba,dc=org",
+                      "objectUUID": b"0123456789abcde3",
+                      "y": ["a", "A"]})
+
+        res = self.ldb.search(expression="(y=a)",
+                              base="dc=samba,dc=org")
+        self.assertEquals(len(res), 2)
+
+        self.ldb.transaction_start()
+
+        self.ldb.add({"dn": "@ATTRIBUTES",
+                      "y": "CASE_INSENSITIVE"})
+
+        self.ldb.transaction_commit()
+
+        # We must still have a working index
+        res = self.ldb.search(expression="(y=a)",
+                              base="dc=samba,dc=org")
+
+        if hasattr(self, 'IDXGUID'):
+            self.assertEquals(len(res), 3)
+        else:
+            # We should not return this entry twice, but sadly
+            # we have not yet fixed
+            # https://bugzilla.samba.org/show_bug.cgi?id=13361
+            self.assertEquals(len(res), 4)
+
+
+    def tearDown(self):
+        super(BadIndexTests, self).tearDown()
+
+
+class GUIDBadIndexTests(BadIndexTests):
+    """Test Bad index things with GUID index mode"""
+    def setUp(self):
+        self.IDXGUID = True
+
+        super(GUIDBadIndexTests, self).setUp()
+
+
 class DnTests(TestCase):
 
     def setUp(self):
-- 
2.11.0


From 7e03a8c1f17cc71b9e64d28a13625813a46eeb68 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:42:17 +1300
Subject: [PATCH 04/54] ldb_wrap: Remove ldb_transaction_cancel_noerr from
 ldb_wrap_fork_hook()

Writing to a TDB, without locks (these are per-process) in a forked child is never going to
end well, if a transaction is open at this point we have bigger problems.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb-samba/ldb_wrap.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 9959b04ed95..8c3bf6f7bf3 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -329,20 +329,11 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 }
 
 /*
-  when we fork() we need to make sure that any open ldb contexts have
-  any open transactions cancelled (ntdb databases doesn't need reopening,
-  as we don't use clear_if_first).
- */
+  call tdb_reopen_all() in case there is a TDB open so we are
+  not blocked from re-opening it inside ldb_tdb.
+*/
  void ldb_wrap_fork_hook(void)
 {
-	struct ldb_wrap *w;
-
-	for (w=ldb_wrap_list; w; w=w->next) {
-		if (ldb_transaction_cancel_noerr(w->ldb) != LDB_SUCCESS) {
-			smb_panic("Failed to cancel child transactions\n");
-		}
-	}
-
 	if (tdb_reopen_all(1) != 0) {
 		smb_panic("tdb_reopen_all failed\n");
 	}
-- 
2.11.0


From 697bafe111fb208a86fa030794ec9288938422c6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:44:52 +1300
Subject: [PATCH 05/54] ldb_wrap: Remove the magic cache of database handles
 except for sam.ldb

sam.ldb is handled in samdb_connect_url(), not this function.

This cache caused issues when "private dir" was changed in a testing script, but also
just generates many-owner shared mutable state that is frowned upon these days.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb-samba/ldb_wrap.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 8c3bf6f7bf3..53255556e7c 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -303,9 +303,13 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 	struct ldb_context *ldb;
 	int ret;
 
-	ldb = ldb_wrap_find(url, ev, lp_ctx, session_info, credentials, flags);
-	if (ldb != NULL)
-		return talloc_reference(mem_ctx, ldb);
+	/*
+	 * Unlike samdb_connect_url() do not try and cache the LDB
+	 * handle, get a new one each time.  Only sam.ldb is
+	 * punitively expensive to open and helpful caches like this
+	 * cause challenges (such as if the value for 'private dir'
+	 * changes).
+	 */
 
 	ldb = samba_ldb_init(mem_ctx, ev, lp_ctx, session_info, credentials);
 
@@ -318,11 +322,6 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 		return NULL;
 	}
 
-	if (!ldb_wrap_add(url, ev, lp_ctx, session_info, credentials, flags, ldb)) {
-		talloc_free(ldb);
-		return NULL;
-	}
-
 	DEBUG(3,("ldb_wrap open of %s\n", url));
 
 	return ldb;
-- 
2.11.0


From 694ece0d6d6386894bed742ae016b54b13f2af1c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 8 Mar 2018 14:01:50 +1300
Subject: [PATCH 06/54] ldb: Fix missing NULL terminator in ldb_mod_op_test
 testsuite

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index b91130252a7..0d38c4f9681 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3099,7 +3099,7 @@ static int ldb_unique_index_test_setup(void **state)
 		"dn: @INDEXLIST\n"
 		"@IDXATTR: cn\n"
 		"\n";
-	const char *options[] = {"modules:unique_index_test"};
+	const char *options[] = {"modules:unique_index_test", NULL};
 
 
 	ret = ldb_register_module(&ldb_unique_index_test_module_ops);
@@ -3201,7 +3201,7 @@ static int ldb_non_unique_index_test_setup(void **state)
 		"dn: @INDEXLIST\n"
 		"@IDXATTR: cn\n"
 		"\n";
-	const char *options[] = {"modules:unique_index_test"};
+	const char *options[] = {"modules:unique_index_test", NULL};
 
 
 	ret = ldb_register_module(&ldb_unique_index_test_module_ops);
-- 
2.11.0


From 063b7b79b6dce1ef189b1b460c6f7a44fbf33258 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 13:05:55 +1300
Subject: [PATCH 07/54] samba-tool domain classicupgrade: Do not mix
 python-samdb transactions and passdb modifications

This worked previously because we knew the same tdb was in use under the hood,
but now that nested TDB transactions are banned this breaks, and it breaks for
LMDB.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/upgrade.py | 48 +++++++++++++++++++-----------------------------
 1 file changed, 19 insertions(+), 29 deletions(-)

diff --git a/python/samba/upgrade.py b/python/samba/upgrade.py
index ff470665fe9..e3fa376e053 100644
--- a/python/samba/upgrade.py
+++ b/python/samba/upgrade.py
@@ -785,38 +785,28 @@ Please fix this account before attempting to upgrade again
     result.samdb.transaction_commit()
 
     logger.info("Adding users")
-    # Start a new transaction (should speed this up a little, due to index churn)
-    result.samdb.transaction_start()
-
-    try:
-        # Export users to samba4 backend
-        logger.info("Importing users")
-        for username in userdata:
-            if username.lower() == 'administrator':
-                if userdata[username].user_sid != dom_sid(str(domainsid) + "-500"):
-                    logger.error("User 'Administrator' in your existing directory has SID %s, expected it to be %s" % (userdata[username].user_sid, dom_sid(str(domainsid) + "-500")))
-                    raise ProvisioningError("User 'Administrator' in your existing directory does not have SID ending in -500")
-            if username.lower() == 'root':
-                if userdata[username].user_sid == dom_sid(str(domainsid) + "-500"):
-                    logger.warn('User root has been replaced by Administrator')
-                else:
-                    logger.warn('User root has been kept in the directory, it should be removed in favour of the Administrator user')
 
-            s4_passdb.add_sam_account(userdata[username])
-            if username in uids:
-                add_ad_posix_idmap_entry(result.samdb, userdata[username].user_sid, uids[username], "ID_TYPE_UID", logger)
-                if (username in homes) and (homes[username] is not None) and \
-                   (username in shells) and (shells[username] is not None) and \
-                   (username in pgids) and (pgids[username] is not None):
-                    add_posix_attrs(samdb=result.samdb, sid=userdata[username].user_sid, name=username, nisdomain=domainname.lower(), xid_type="ID_TYPE_UID", home=homes[username], shell=shells[username], pgid=pgids[username], logger=logger)
+    # Export users to samba4 backend
+    logger.info("Importing users")
+    for username in userdata:
+        if username.lower() == 'administrator':
+            if userdata[username].user_sid != dom_sid(str(domainsid) + "-500"):
+                logger.error("User 'Administrator' in your existing directory has SID %s, expected it to be %s" % (userdata[username].user_sid, dom_sid(str(domainsid) + "-500")))
+                raise ProvisioningError("User 'Administrator' in your existing directory does not have SID ending in -500")
+        if username.lower() == 'root':
+            if userdata[username].user_sid == dom_sid(str(domainsid) + "-500"):
+                logger.warn('User root has been replaced by Administrator')
+            else:
+                logger.warn('User root has been kept in the directory, it should be removed in favour of the Administrator user')
 
-    except:
-        # We need this, so that we do not give even more errors due to not cancelling the transaction
-        result.samdb.transaction_cancel()
-        raise
+        s4_passdb.add_sam_account(userdata[username])
+        if username in uids:
+            add_ad_posix_idmap_entry(result.samdb, userdata[username].user_sid, uids[username], "ID_TYPE_UID", logger)
+            if (username in homes) and (homes[username] is not None) and \
+               (username in shells) and (shells[username] is not None) and \
+               (username in pgids) and (pgids[username] is not None):
+                add_posix_attrs(samdb=result.samdb, sid=userdata[username].user_sid, name=username, nisdomain=domainname.lower(), xid_type="ID_TYPE_UID", home=homes[username], shell=shells[username], pgid=pgids[username], logger=logger)
 
-    logger.info("Committing 'add users' transaction to disk")
-    result.samdb.transaction_commit()
 
     logger.info("Adding users to groups")
     # Start a new transaction (should speed this up a little, due to index churn)
-- 
2.11.0


From e97a09eabff698a4e5ec12434f7ee2cb266bd393 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 16 Feb 2018 17:13:26 +1300
Subject: [PATCH 08/54] ldb: Change some prototypes to using ldb_val instead of
 TDB_DATA

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_index.c |  8 ++++----
 lib/ldb/ldb_tdb/ldb_tdb.c   | 49 +++++++++++++++++++++++++++++++++++++--------
 lib/ldb/ldb_tdb/ldb_tdb.h   |  8 ++++----
 3 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index bb534c3833c..076db10f2dd 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2783,11 +2783,11 @@ static int re_key(struct ltdb_private *ltdb, struct ldb_val ldb_key, struct ldb_
 	}
 	if (key.dsize != key2.dsize ||
 	    (memcmp(key.dptr, key2.dptr, key.dsize) != 0)) {
-		TDB_DATA data = {
-			.dptr = val.data,
-			.dsize = val.length
+		struct ldb_val ldb_key2 = {
+			.data = key2.dptr,
+			.length = key2.dsize
 		};
-		ltdb->kv_ops->update_in_iterate(ltdb, key, key2, data, ctx);
+		ltdb->kv_ops->update_in_iterate(ltdb, ldb_key, ldb_key2, val, ctx);
 	}
 	talloc_free(key2.dptr);
 
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 20796f2162d..f94a308cbf9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -417,8 +417,17 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
 	return ret;
 }
 
-static int ltdb_tdb_store(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags)
+static int ltdb_tdb_store(struct ltdb_private *ltdb, struct ldb_val ldb_key,
+			  struct ldb_val ldb_data, int flags)
 {
+	TDB_DATA key = {
+		.dptr = ldb_key.data,
+		.dsize = ldb_key.length
+	};
+	TDB_DATA data = {
+		.dptr = ldb_data.data,
+		.dsize = ldb_data.length
+	};
 	return tdb_store(ltdb->tdb, key, data, flags);
 }
 
@@ -439,7 +448,8 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
 {
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
-	TDB_DATA tdb_key, tdb_data;
+	TDB_DATA tdb_key;
+	struct ldb_val ldb_key;
 	struct ldb_val ldb_data;
 	int ret = LDB_SUCCESS;
 	TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -465,10 +475,10 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
 		return LDB_ERR_OTHER;
 	}
 
-	tdb_data.dptr = ldb_data.data;
-	tdb_data.dsize = ldb_data.length;
+	ldb_key.data = tdb_key.dptr;
+	ldb_key.length = tdb_key.dsize;
 
-	ret = ltdb->kv_ops->store(ltdb, tdb_key, tdb_data, flgs);
+	ret = ltdb->kv_ops->store(ltdb, ldb_key, ldb_data, flgs);
 	if (ret != 0) {
 		bool is_special = ldb_dn_is_special(msg->dn);
 		ret = ltdb->kv_ops->error(ltdb);
@@ -654,8 +664,12 @@ static int ltdb_add(struct ltdb_context *ctx)
 	return ret;
 }
 
-static int ltdb_tdb_delete(struct ltdb_private *ltdb, TDB_DATA tdb_key)
+static int ltdb_tdb_delete(struct ltdb_private *ltdb, struct ldb_val ldb_key)
 {
+	TDB_DATA tdb_key = {
+		.dptr = ldb_key.data,
+		.dsize = ldb_key.length
+	};
 	return tdb_delete(ltdb->tdb, tdb_key);
 }
 
@@ -668,6 +682,7 @@ int ltdb_delete_noindex(struct ldb_module *module,
 {
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+	struct ldb_val ldb_key;
 	TDB_DATA tdb_key;
 	int ret;
 	TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -686,7 +701,10 @@ int ltdb_delete_noindex(struct ldb_module *module,
 		return LDB_ERR_OTHER;
 	}
 
-	ret = ltdb->kv_ops->delete(ltdb, tdb_key);
+	ldb_key.data = tdb_key.dptr;
+	ldb_key.length = tdb_key.dsize;
+
+	ret = ltdb->kv_ops->delete(ltdb, ldb_key);
 	TALLOC_FREE(tdb_key_ctx);
 
 	if (ret != 0) {
@@ -1777,12 +1795,27 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
 	}
 }
 
-static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA key2, TDB_DATA data, void *state)
+static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
+				      struct ldb_val ldb_key,
+				      struct ldb_val ldb_key2,
+				      struct ldb_val ldb_data, void *state)
 {
 	int tdb_ret;
 	struct ldb_context *ldb;
 	struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
 	struct ldb_module *module = ctx->module;
+	TDB_DATA key = {
+		.dptr = ldb_key.data,
+		.dsize = ldb_key.length
+	};
+	TDB_DATA key2 = {
+		.dptr = ldb_key2.data,
+		.dsize = ldb_key2.length
+	};
+	TDB_DATA data = {
+		.dptr = ldb_data.data,
+		.dsize = ldb_data.length
+	};
 
 	ldb = ldb_module_get_ctx(module);
 
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 28dd20915c7..2c0ea7cfe38 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -10,11 +10,11 @@ typedef int (*ldb_kv_traverse_fn)(struct ltdb_private *ltdb,
 				  void *ctx);
 
 struct kv_db_ops {
-	int (*store)(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags);
-	int (*delete)(struct ltdb_private *ltdb, TDB_DATA key);
+	int (*store)(struct ltdb_private *ltdb, struct ldb_val key, struct ldb_val data, int flags);
+	int (*delete)(struct ltdb_private *ltdb, struct ldb_val key);
 	int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
-	int (*update_in_iterate)(struct ltdb_private *ltdb, TDB_DATA key,
-				 TDB_DATA key2, TDB_DATA data, void *ctx);
+	int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
+				 struct ldb_val key2, struct ldb_val data, void *ctx);
 	int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
                                int (*parser)(TDB_DATA key, TDB_DATA data,
                                              void *private_data),
-- 
2.11.0


From 9b363597f46c458f83ed0f10980a1a2ce1f08cf9 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 19 Feb 2018 12:37:20 +1300
Subject: [PATCH 09/54] ldb: Change remaining fetch prototypes to remove
 TDB_DATA

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_search.c | 28 +++++++++++++++-------------
 lib/ldb/ldb_tdb/ldb_tdb.c    | 38 +++++++++++++++++++++++++++++++++++---
 lib/ldb/ldb_tdb/ldb_tdb.h    |  4 ++--
 3 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 78ef8b0abb1..35583a1eaa3 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -180,17 +180,15 @@ struct ltdb_parse_data_unpack_ctx {
 	unsigned int unpack_flags;
 };
 
-static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
+static int ltdb_parse_data_unpack(struct ldb_val key,
+				  struct ldb_val data,
 				  void *private_data)
 {
 	struct ltdb_parse_data_unpack_ctx *ctx = private_data;
 	unsigned int nb_elements_in_db;
 	int ret;
 	struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
-	struct ldb_val data_parse = {
-		.data = data.dptr,
-		.length = data.dsize
-	};
+	struct ldb_val data_parse = data;
 
 	if (ctx->unpack_flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
 		/*
@@ -200,13 +198,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
 		 * and the caller needs a stable result.
 		 */
 		data_parse.data = talloc_memdup(ctx->msg,
-						data.dptr,
-						data.dsize);
+						data.data,
+						data.length);
 		if (data_parse.data == NULL) {
 			ldb_debug(ldb, LDB_DEBUG_ERROR,
 				  "Unable to allocate data(%d) for %*.*s\n",
-				  (int)data.dsize,
-				  (int)key.dsize, (int)key.dsize, key.dptr);
+				  (int)data.length,
+				  (int)key.length, (int)key.length, key.data);
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 	}
@@ -217,13 +215,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
 						   ctx->unpack_flags,
 						   &nb_elements_in_db);
 	if (ret == -1) {
-		if (data_parse.data != data.dptr) {
+		if (data_parse.data != data.data) {
 			talloc_free(data_parse.data);
 		}
 
 		ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %*.*s\n",
-			  (int)key.dsize, (int)key.dsize, key.dptr);
-		return LDB_ERR_OPERATIONS_ERROR;		
+			  (int)key.length, (int)key.length, key.data);
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 	return ret;
 }
@@ -246,13 +244,17 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
 		.module = module,
 		.unpack_flags = unpack_flags
 	};
+	struct ldb_val ldb_key = {
+		.data = tdb_key.dptr,
+		.length = tdb_key.dsize
+	};
 
 	memset(msg, 0, sizeof(*msg));
 
 	msg->num_elements = 0;
 	msg->elements = NULL;
 
-	ret = ltdb->kv_ops->fetch_and_parse(ltdb, tdb_key,
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, ldb_key,
 					    ltdb_parse_data_unpack, &ctx);
 
 	if (ret == -1) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f94a308cbf9..423b5a1f687 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1765,6 +1765,9 @@ struct kv_ctx {
 	ldb_kv_traverse_fn kv_traverse_fn;
 	void *ctx;
 	struct ltdb_private *ltdb;
+	int (*parser)(struct ldb_val key,
+		      struct ldb_val data,
+		      void *private_data);
 };
 
 static int ldb_tdb_traverse_fn_wrapper(struct tdb_context *tdb, TDB_DATA tdb_key, TDB_DATA tdb_data, void *ctx)
@@ -1847,12 +1850,41 @@ static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
 	return tdb_ret;
 }
 
-static int ltdb_tdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
-				 int (*parser)(TDB_DATA key, TDB_DATA data,
+static int ltdb_tdb_parse_record_wrapper(TDB_DATA tdb_key, TDB_DATA tdb_data,
+					 void *ctx)
+{
+	struct kv_ctx *kv_ctx = ctx;
+	struct ldb_val key = {
+		.length = tdb_key.dsize,
+		.data = tdb_key.dptr,
+	};
+	struct ldb_val data = {
+		.length = tdb_data.dsize,
+		.data = tdb_data.dptr,
+	};
+
+	return kv_ctx->parser(key, data, kv_ctx->ctx);
+}
+
+static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
+				 struct ldb_val ldb_key,
+				 int (*parser)(struct ldb_val key,
+					       struct ldb_val data,
 					       void *private_data),
 				 void *ctx)
 {
-	return tdb_parse_record(ltdb->tdb, key, parser, ctx);
+	struct kv_ctx kv_ctx = {
+		.parser = parser,
+		.ctx = ctx,
+		.ltdb = ltdb
+	};
+	TDB_DATA key = {
+		.dptr = ldb_key.data,
+		.dsize = ldb_key.length
+	};
+
+	return tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
+				&kv_ctx);
 }
 
 static const char * ltdb_tdb_name(struct ltdb_private *ltdb)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 2c0ea7cfe38..f14666ba88a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -15,8 +15,8 @@ struct kv_db_ops {
 	int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
 	int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
 				 struct ldb_val key2, struct ldb_val data, void *ctx);
-	int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
-                               int (*parser)(TDB_DATA key, TDB_DATA data,
+	int (*fetch_and_parse)(struct ltdb_private *ltdb, struct ldb_val key,
+                               int (*parser)(struct ldb_val key, struct ldb_val data,
                                              void *private_data),
                                void *ctx);
 	int (*lock_read)(struct ldb_module *);
-- 
2.11.0


From 4ed37516f5e785288be53d2c54a6fc962a1ab759 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 17 Jan 2018 14:02:09 +1300
Subject: [PATCH 10/54] upgradeprovision: Do not copy backup lmdb -lock files

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/scripting/bin/samba_upgradeprovision | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/source4/scripting/bin/samba_upgradeprovision b/source4/scripting/bin/samba_upgradeprovision
index f3e690ba552..e0db12e0818 100755
--- a/source4/scripting/bin/samba_upgradeprovision
+++ b/source4/scripting/bin/samba_upgradeprovision
@@ -1397,9 +1397,10 @@ def backup_provision(paths, dir, only_db):
     else:
         os.mkdir(os.path.join(dir, "sam.ldb.d"), 0700)
 
-        for ldb in os.listdir(samldbdir):
-            tdb_util.tdb_copy(os.path.join(samldbdir, ldb),
-                              os.path.join(dir, "sam.ldb.d", ldb))
+        for ldb_name in os.listdir(samldbdir):
+            if not ldb_name.endswith("-lock"):
+                tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
+                                  os.path.join(dir, "sam.ldb.d", ldb_name))
 
 
 def sync_calculated_attributes(samdb, names):
-- 
2.11.0


From a8a15fdbe5d609868e94649bb083dcd811b5f18e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 6 Mar 2018 17:00:07 +1300
Subject: [PATCH 11/54] ldb: Ignore these tests in mdb test mode

These are tests are specifically for when the GUID index is not in use
which is always in with ldb_mdb.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 0d38c4f9681..db0c594b430 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3389,6 +3389,11 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
 	char *debug_string = NULL;
 	char *p = NULL;
 
+	/* The non-GUID mode is not compatible with mdb */
+	if (strcmp(TEST_BE, "mdb") == 0) {
+		return;
+	}
+	
 	ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
 	tmp_ctx = talloc_new(test_ctx);
 	assert_non_null(tmp_ctx);
@@ -3438,6 +3443,11 @@ static void test_ldb_duplicate_dn_logging(void **state)
 	TALLOC_CTX *tmp_ctx;
 	char *debug_string = NULL;
 
+	/* The non-GUID mode is not compatible with mdb */
+	if (strcmp(TEST_BE, "mdb") == 0) {
+		return;
+	}
+	
 	ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
 	tmp_ctx = talloc_new(test_ctx);
 	assert_non_null(tmp_ctx);
@@ -3836,6 +3846,7 @@ int main(int argc, const char **argv)
 			test_ldb_add_to_index_unique_values_required,
 			ldb_non_unique_index_test_setup,
 			ldb_non_unique_index_test_teardown),
+		/* These tests are not compatible with mdb */
 		cmocka_unit_test_setup_teardown(
 			test_ldb_unique_index_duplicate_logging,
 			ldb_unique_index_test_setup,
-- 
2.11.0


From 2d21aeda1ea19b653e8ebdf900105d3dce063ecb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 11:10:25 +1300
Subject: [PATCH 12/54] ldb: Allow GUID index mode to be tested on TDB

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 149 ++++++++++++++++++++++++++++++++++------
 lib/ldb/wscript                 |   7 ++
 2 files changed, 136 insertions(+), 20 deletions(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index db0c594b430..dfc0209260e 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -202,6 +202,16 @@ static void test_ldif_message_redacted(void **state)
 static int ldbtest_setup(void **state)
 {
 	struct ldbtest_ctx *test_ctx;
+	struct ldb_ldif *ldif;
+#ifdef GUID_IDX
+	const char *index_ldif =		\
+		"dn: @INDEXLIST\n"
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+		"\n";
+#else
+	const char *index_ldif = "\n";
+#endif
 	int ret;
 
 	ldbtest_noconn_setup((void **) &test_ctx);
@@ -209,6 +219,10 @@ static int ldbtest_setup(void **state)
 	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
 	assert_int_equal(ret, 0);
 
+	while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+		ret = ldb_add(test_ctx->ldb, ldif->msg);
+		assert_int_equal(ret, LDB_SUCCESS);
+	}
 	*state = test_ctx;
 	return 0;
 }
@@ -241,6 +255,9 @@ static void test_ldb_add(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
@@ -279,6 +296,9 @@ static void test_ldb_search(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val1");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcde1");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
@@ -294,6 +314,9 @@ static void test_ldb_search(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val2");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcde2");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
@@ -390,7 +413,8 @@ static void assert_dn_doesnt_exist(struct ldbtest_ctx *test_ctx,
 
 static void add_dn_with_cn(struct ldbtest_ctx *test_ctx,
 			   struct ldb_dn *dn,
-			   const char *cn_value)
+			   const char *cn_value,
+			   const char *uuid_value)
 {
 	int ret;
 	TALLOC_CTX *tmp_ctx;
@@ -409,6 +433,9 @@ static void add_dn_with_cn(struct ldbtest_ctx *test_ctx,
 	ret = ldb_msg_add_string(msg, "cn", cn_value);
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", uuid_value);
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -428,7 +455,9 @@ static void test_ldb_del(void **state)
 	dn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "%s", basedn);
 	assert_non_null(dn);
 
-	add_dn_with_cn(test_ctx, dn, "test_del_cn_val");
+	add_dn_with_cn(test_ctx, dn,
+		       "test_del_cn_val",
+		       "0123456789abcdef");
 
 	ret = ldb_delete(test_ctx->ldb, dn);
 	assert_int_equal(ret, LDB_SUCCESS);
@@ -552,7 +581,8 @@ static void test_ldb_build_search_req(void **state)
 
 static void add_keyval(struct ldbtest_ctx *test_ctx,
 		       const char *key,
-		       const char *val)
+		       const char *val,
+		       const char *uuid)
 {
 	int ret;
 	struct ldb_message *msg;
@@ -566,6 +596,9 @@ static void add_keyval(struct ldbtest_ctx *test_ctx,
 	ret = ldb_msg_add_string(msg, key, val);
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", uuid);
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
@@ -601,7 +634,8 @@ static void test_transactions(void **state)
 	ret = ldb_transaction_start(test_ctx->ldb);
 	assert_int_equal(ret, 0);
 
-	add_keyval(test_ctx, "vegetable", "carrot");
+	add_keyval(test_ctx, "vegetable", "carrot",
+		   "0123456789abcde0");
 
 	/* commit lev-0 transaction */
 	ret = ldb_transaction_commit(test_ctx->ldb);
@@ -611,7 +645,8 @@ static void test_transactions(void **state)
 	ret = ldb_transaction_start(test_ctx->ldb);
 	assert_int_equal(ret, 0);
 
-	add_keyval(test_ctx, "fruit", "apple");
+	add_keyval(test_ctx, "fruit", "apple",
+		   "0123456789abcde1");
 
 	/* abort lev-1 nested transaction */
 	ret = ldb_transaction_cancel(test_ctx->ldb);
@@ -637,14 +672,16 @@ static void test_nested_transactions(void **state)
 	ret = ldb_transaction_start(test_ctx->ldb);
 	assert_int_equal(ret, 0);
 
-	add_keyval(test_ctx, "vegetable", "carrot");
+	add_keyval(test_ctx, "vegetable", "carrot",
+		   "0123456789abcde0");
 
 
 	/* start another lev-1 nested transaction */
 	ret = ldb_transaction_start(test_ctx->ldb);
 	assert_int_equal(ret, 0);
 
-	add_keyval(test_ctx, "fruit", "apple");
+	add_keyval(test_ctx, "fruit", "apple",
+		   "0123456789abcde1");
 
 	/* abort lev-1 nested transaction */
 	ret = ldb_transaction_cancel(test_ctx->ldb);
@@ -825,6 +862,7 @@ static int ldb_modify_test_setup(void **state)
 	struct ldb_mod_test_ctx *mod_test_ctx;
 	struct keyval kvs[] = {
 		{ "cn", "test_mod_cn" },
+		{ "objectUUID", "0123456789abcdef"},
 		{ NULL, NULL },
 	};
 
@@ -1129,6 +1167,7 @@ static int ldb_search_test_setup(void **state)
 		{ "cn", "test_search_cn2" },
 		{ "uid", "test_search_uid" },
 		{ "uid", "test_search_uid2" },
+		{ "objectUUID", "0123456789abcde0"},
 		{ NULL, NULL },
 	};
 	struct keyval kvs2[] = {
@@ -1136,6 +1175,7 @@ static int ldb_search_test_setup(void **state)
 		{ "cn", "test_search_2_cn2" },
 		{ "uid", "test_search_2_uid" },
 		{ "uid", "test_search_2_uid2" },
+		{ "objectUUID", "0123456789abcde1"},
 		{ NULL, NULL },
 	};
 
@@ -1532,6 +1572,12 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
 			exit(LDB_ERR_OPERATIONS_ERROR);
 		}
 
+		ret = ldb_msg_add_string(msg, "objectUUID",
+					 "0123456789abcdef");
+		if (ret != 0) {
+			exit(ret);
+		}
+
 		ret = ldb_add(ctx->test_ctx->ldb, msg);
 		if (ret != 0) {
 			exit(ret);
@@ -1894,13 +1940,16 @@ static void test_ldb_modify_during_search(void **state, bool add_index,
 
 		ret = ldb_msg_add_string(msg, "@IDXATTR", "cn");
 		assert_int_equal(ret, LDB_SUCCESS);
-
 		ret = ldb_add(search_test_ctx->ldb_test_ctx->ldb,
 			      msg);
-
+		if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+			msg->elements[0].flags = LDB_FLAG_MOD_ADD;
+			ret = ldb_modify(search_test_ctx->ldb_test_ctx->ldb,
+					 msg);
+		}
 		assert_int_equal(ret, LDB_SUCCESS);
 	}
-
+		
 	tevent_loop_allow_nesting(search_test_ctx->ldb_test_ctx->ev);
 
 	ctx.basedn
@@ -2436,6 +2485,7 @@ static int ldb_case_test_setup(void **state)
 	struct keyval kvs[] = {
 		{ "cn", "CaseInsensitiveValue" },
 		{ "uid", "CaseSensitiveValue" },
+		{ "objectUUID", "0123456789abcdef" },
 		{ NULL, NULL },
 	};
 
@@ -2649,6 +2699,11 @@ static void test_ldb_attrs_index_handler(void **state)
 	/* Add the index (actually any modify will do) */
 	while ((ldif = ldb_ldif_read_string(ldb_test_ctx->ldb, &index_ldif))) {
 		ret = ldb_add(ldb_test_ctx->ldb, ldif->msg);
+		if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+			ldif->msg->elements[0].flags = LDB_FLAG_MOD_ADD;
+			ret = ldb_modify(ldb_test_ctx->ldb,
+					 ldif->msg);
+		}
 		assert_int_equal(ret, LDB_SUCCESS);
 	}
 
@@ -2734,7 +2789,8 @@ static int ldb_rename_test_setup(void **state)
 
 	add_dn_with_cn(ldb_test_ctx,
 		       rename_test_ctx->basedn,
-		       "test_rename_cn_val");
+		       "test_rename_cn_val",
+		       "0123456789abcde0");
 
 	*state = rename_test_ctx;
 	return 0;
@@ -2839,7 +2895,8 @@ static void test_ldb_rename_to_exists(void **state)
 
 	add_dn_with_cn(rename_test_ctx->ldb_test_ctx,
 		       new_dn,
-		       "test_rename_cn_val");
+		       "test_rename_cn_val",
+		       "0123456789abcde1");
 
 	ret = ldb_rename(rename_test_ctx->ldb_test_ctx->ldb,
 			 rename_test_ctx->basedn,
@@ -3006,6 +3063,10 @@ static void test_read_only(void **state)
 		ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 		assert_int_equal(ret, 0);
 
+		ret = ldb_msg_add_string(msg, "objectUUID",
+					 "0123456789abcde1");
+		assert_int_equal(ret, LDB_SUCCESS);
+
 		ret = ldb_add(ro_ldb, msg);
 		assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
 		TALLOC_FREE(msg);
@@ -3025,6 +3086,10 @@ static void test_read_only(void **state)
 		ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 		assert_int_equal(ret, 0);
 
+		ret = ldb_msg_add_string(msg, "objectUUID",
+					 "0123456789abcde2");
+		assert_int_equal(ret, LDB_SUCCESS);
+
 		ret = ldb_add(rw_ldb, msg);
 		assert_int_equal(ret, LDB_SUCCESS);
 		TALLOC_FREE(msg);
@@ -3098,6 +3163,10 @@ static int ldb_unique_index_test_setup(void **state)
 	const char *index_ldif =  \
 		"dn: @INDEXLIST\n"
 		"@IDXATTR: cn\n"
+#ifdef GUID_IDX
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+#endif
 		"\n";
 	const char *options[] = {"modules:unique_index_test", NULL};
 
@@ -3186,6 +3255,10 @@ static void test_ldb_add_unique_value_to_unique_index(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg, "objectUUID",
+				 "0123456789abcde1");
+	assert_int_equal(ret, LDB_SUCCESS);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3200,6 +3273,10 @@ static int ldb_non_unique_index_test_setup(void **state)
 	const char *index_ldif =  \
 		"dn: @INDEXLIST\n"
 		"@IDXATTR: cn\n"
+#ifdef GUID_IDX
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+#endif
 		"\n";
 	const char *options[] = {"modules:unique_index_test", NULL};
 
@@ -3270,6 +3347,10 @@ static void test_ldb_add_duplicate_value_to_unique_index(void **state)
 	ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg01, "objectUUID",
+				 "0123456789abcde1");
+	assert_int_equal(ret, LDB_SUCCESS);
+
 	ret = ldb_add(test_ctx->ldb, msg01);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3281,6 +3362,10 @@ static void test_ldb_add_duplicate_value_to_unique_index(void **state)
 
 	ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
+	
+	ret = ldb_msg_add_string(msg02, "objectUUID",
+				 "0123456789abcde2");
+	assert_int_equal(ret, LDB_SUCCESS);
 
 	ret = ldb_add(test_ctx->ldb, msg02);
 	assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
@@ -3311,6 +3396,9 @@ static void test_ldb_add_to_index_duplicates_allowed(void **state)
 	ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg01, "objectUUID",
+				 "0123456789abcde1");
+
 	ret = ldb_add(test_ctx->ldb, msg01);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3323,6 +3411,9 @@ static void test_ldb_add_to_index_duplicates_allowed(void **state)
 	ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg02, "objectUUID",
+				 "0123456789abcde2");
+	
 	ret = ldb_add(test_ctx->ldb, msg02);
 	assert_int_equal(ret, LDB_SUCCESS);
 	talloc_free(tmp_ctx);
@@ -3352,6 +3443,9 @@ static void test_ldb_add_to_index_unique_values_required(void **state)
 	ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg01, "objectUUID",
+				 "0123456789abcde1");
+	
 	ret = ldb_add(test_ctx->ldb, msg01);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3364,6 +3458,9 @@ static void test_ldb_add_to_index_unique_values_required(void **state)
 	ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg02, "objectUUID",
+				 "0123456789abcde2");
+	
 	ret = ldb_add(test_ctx->ldb, msg02);
 	assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
 	talloc_free(tmp_ctx);
@@ -3389,10 +3486,10 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
 	char *debug_string = NULL;
 	char *p = NULL;
 
-	/* The non-GUID mode is not compatible with mdb */
-	if (strcmp(TEST_BE, "mdb") == 0) {
-		return;
-	}
+	/* The GUID mode is not compatible with this test */
+#ifdef GUID_IDX
+	return;
+#endif
 	
 	ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
 	tmp_ctx = talloc_new(test_ctx);
@@ -3407,6 +3504,9 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
 	ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg01, "objectUUID",
+				 "0123456789abcde1");
+	
 	ret = ldb_add(test_ctx->ldb, msg01);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3419,6 +3519,9 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
 	ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg02, "objectUUID",
+				 "0123456789abcde2");
+	
 	ret = ldb_add(test_ctx->ldb, msg02);
 	assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
 
@@ -3443,10 +3546,10 @@ static void test_ldb_duplicate_dn_logging(void **state)
 	TALLOC_CTX *tmp_ctx;
 	char *debug_string = NULL;
 
-	/* The non-GUID mode is not compatible with mdb */
-	if (strcmp(TEST_BE, "mdb") == 0) {
-		return;
-	}
+	/* The GUID mode is not compatible with this test */
+#ifdef GUID_IDX
+	return;
+#endif
 	
 	ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
 	tmp_ctx = talloc_new(test_ctx);
@@ -3461,6 +3564,9 @@ static void test_ldb_duplicate_dn_logging(void **state)
 	ret = ldb_msg_add_string(msg01, "cn", "test_unique_index01");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg01, "objectUUID",
+				 "0123456789abcde1");
+	
 	ret = ldb_add(test_ctx->ldb, msg01);
 	assert_int_equal(ret, LDB_SUCCESS);
 
@@ -3473,6 +3579,9 @@ static void test_ldb_duplicate_dn_logging(void **state)
 	ret = ldb_msg_add_string(msg02, "cn", "test_unique_index02");
 	assert_int_equal(ret, LDB_SUCCESS);
 
+	ret = ldb_msg_add_string(msg02, "objectUUID",
+				 "0123456789abcde2");
+	
 	ret = ldb_add(test_ctx->ldb, msg02);
 	assert_int_equal(ret, LDB_ERR_ENTRY_ALREADY_EXISTS);
 
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 1455f92eb2e..f959d59b28f 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -353,6 +353,12 @@ def build(bld):
                          deps='cmocka ldb',
                          install=False)
 
+        bld.SAMBA_BINARY('ldb_tdb_guid_mod_op_test',
+                         source='tests/ldb_mod_op_test.c',
+                         cflags='-DTEST_BE=\"tdb\" -DGUID_IDX=1',
+                         deps='cmocka ldb',
+                         install=False)
+
         bld.SAMBA_BINARY('ldb_msg_test',
                          source='tests/ldb_msg.c',
                          deps='cmocka ldb',
@@ -386,6 +392,7 @@ def test(ctx):
 
     cmocka_ret = 0
     for test_exe in ['ldb_tdb_mod_op_test',
+                     'ldb_tdb_guid_mod_op_test',
                      'ldb_msg_test']:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
             cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
-- 
2.11.0


From 5c090049d5d81168fd264ed363810dc53cdaa2c5 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:10:54 +1300
Subject: [PATCH 13/54] ldb index: Fix truncation key length calculation

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_index.c   | 19 +++++++++++++++++--
 lib/ldb/tests/python/index.py |  7 +++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 076db10f2dd..ed28c4f642f 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -848,6 +848,19 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 	unsigned indx_len = 0;
 	unsigned frmt_len = 0;
 
+	if (max_key_length < 4) {
+		ldb_asprintf_errstring(
+			ldb,
+			__location__ ": max_key_length of (%u) < 4",
+			max_key_length);
+		return NULL;
+	}
+	/*
+	 * ltdb_key_dn() makes something 4 bytes longer, it adds a leading
+	 * "DN=" and a trailing string terninator
+	 */
+	max_key_length -= 4;
+
 	if (attr[0] == '@') {
 		attr_for_dn = attr;
 		v = *value;
@@ -911,12 +924,13 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 	if (should_b64_encode) {
 		unsigned vstr_len = 0;
 		char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
+		unsigned num_separators = 3;
 		if (!vstr) {
 			talloc_free(attr_folded);
 			return NULL;
 		}
 		vstr_len = strlen(vstr);
-		key_len = 3 + indx_len + attr_len + vstr_len;
+		key_len = num_separators + indx_len + attr_len + vstr_len;
 		if (key_len > max_key_length) {
 			unsigned excess = key_len - max_key_length;
 			frmt_len = vstr_len - excess;
@@ -943,7 +957,8 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 		}
 		talloc_free(vstr);
 	} else {
-		key_len = 2 + indx_len + attr_len + (int)v.length;
+		unsigned num_separators = 2;
+		key_len = num_separators + indx_len + attr_len + (int)v.length;
 		if (key_len > max_key_length) {
 			unsigned excess = key_len - max_key_length;
 			frmt_len = v.length - excess;
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index e9eaaf059e1..1d66ee930d1 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -113,11 +113,14 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
         super(MaxIndexKeyLengthTests, self).setUp()
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "key_len_test.ldb")
-        # Note that the maximum key length is set to 50
+        # Note that the maximum key length is set to 54
+        # This accounts for the 4 bytes added by the dn formatting
+        # a leading dn=, and a trailing zero terminator
+        #
         self.l = ldb.Ldb(self.url(),
                          options=[
                              "modules:rdn_name",
-                             "max_key_len_for_self_test:50"])
+                             "max_key_len_for_self_test:54"])
         self.l.add({"dn": "@ATTRIBUTES",
                     "uniqueThing": "UNIQUE_INDEX"})
         self.l.add({"dn": "@INDEXLIST",
-- 
2.11.0


From 9ffec8afc419fe26b27e78eb6bd2ccaa46d458bf Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 4 Apr 2018 14:00:57 +1200
Subject: [PATCH 14/54] ldb_tdb: A more robust check for if we can fit the
 index string in

This avoids magic numbers and also is careful against overflow
from a long attr_for_dn.

This is done as a distinct commit to make the previous behaviour
change more clear, and to show that this does not change the
calculations, only improves the overflow check.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_index.c | 72 ++++++++++++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 24 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index ed28c4f642f..6b5c9a800d6 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -842,25 +842,18 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 	const char *attr_for_dn = NULL;
 	int r;
 	bool should_b64_encode;
-	unsigned max_key_length = ltdb_max_key_length(ltdb);
-	unsigned key_len = 0;
-	unsigned attr_len = 0;
-	unsigned indx_len = 0;
-	unsigned frmt_len = 0;
-
-	if (max_key_length < 4) {
-		ldb_asprintf_errstring(
-			ldb,
-			__location__ ": max_key_length of (%u) < 4",
-			max_key_length);
-		return NULL;
-	}
-	/*
-	 * ltdb_key_dn() makes something 4 bytes longer, it adds a leading
-	 * "DN=" and a trailing string terninator
-	 */
-	max_key_length -= 4;
 
+	unsigned int max_key_length = ltdb_max_key_length(ltdb);
+	size_t key_len = 0;
+	size_t attr_len = 0;
+	const size_t indx_len = sizeof(LTDB_INDEX) - 1;
+	unsigned frmt_len = 0;
+	const size_t additional_key_length = 4;
+	unsigned int num_separators = 3; /* Estimate for overflow check */
+	const size_t min_data = 1;
+	const size_t min_key_length = additional_key_length
+		+ indx_len + num_separators + min_data;
+	
 	if (attr[0] == '@') {
 		attr_for_dn = attr;
 		v = *value;
@@ -897,7 +890,29 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 		}
 	}
 	attr_len = strlen(attr_for_dn);
-	indx_len = strlen(LTDB_INDEX);
+
+	/*
+	 * Check if there is any hope this will fit into the DB.
+	 * Overflow here is not actually critical the code below
+	 * checks again to make the printf and the DB does another
+	 * check for too long keys
+	 */
+	if (max_key_length - attr_len < min_key_length) {
+		ldb_asprintf_errstring(
+			ldb,
+			__location__ ": max_key_length "
+			"is too small (%u) < (%u)",
+			max_key_length,
+			(unsigned)(min_key_length + attr_len));
+		talloc_free(attr_folded);
+		return NULL;
+	}
+	
+	/*
+	 * ltdb_key_dn() makes something 4 bytes longer, it adds a leading
+	 * "DN=" and a trailing string terninator
+	 */
+	max_key_length -= additional_key_length;
 
 	/*
 	 * We do not base 64 encode a DN in a key, it has already been
@@ -922,17 +937,20 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 	}
 
 	if (should_b64_encode) {
-		unsigned vstr_len = 0;
+		size_t vstr_len = 0;
 		char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
-		unsigned num_separators = 3;
 		if (!vstr) {
 			talloc_free(attr_folded);
 			return NULL;
 		}
 		vstr_len = strlen(vstr);
+		/* 
+		 * Overflow here is not critical as we only use this
+		 * to choose the printf truncation
+		 */
 		key_len = num_separators + indx_len + attr_len + vstr_len;
 		if (key_len > max_key_length) {
-			unsigned excess = key_len - max_key_length;
+			size_t excess = key_len - max_key_length;
 			frmt_len = vstr_len - excess;
 			*truncation = KEY_TRUNCATED;
 			/*
@@ -957,10 +975,16 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 		}
 		talloc_free(vstr);
 	} else {
-		unsigned num_separators = 2;
+		/* Only need two seperators */
+		num_separators = 2;
+		
+		/* 
+		 * Overflow here is not critical as we only use this
+		 * to choose the printf truncation
+		 */
 		key_len = num_separators + indx_len + attr_len + (int)v.length;
 		if (key_len > max_key_length) {
-			unsigned excess = key_len - max_key_length;
+			size_t excess = key_len - max_key_length;
 			frmt_len = v.length - excess;
 			*truncation = KEY_TRUNCATED;
 			/*
-- 
2.11.0


From b017f4c3d2163182b05550241ac31c9eb8eeb4cb Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:45:28 +1300
Subject: [PATCH 15/54] ldb index: Add tests for truncated base 64 index keys

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/index.py | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 1d66ee930d1..9b9e4f3469f 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -124,7 +124,12 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
         self.l.add({"dn": "@ATTRIBUTES",
                     "uniqueThing": "UNIQUE_INDEX"})
         self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"uniqueThing", b"notUnique"],
+                    "@IDXATTR": [
+                        b"uniqueThing",
+                        b"notUnique",
+                        b"base64____lt",
+                        b"base64_____eq",
+                        b"base64______gt"],
                     "@IDXONE": [b"1"],
                     "@IDXGUID": [b"objectUUID"],
                     "@IDX_DN_GUID": [b"GUID"]})
@@ -870,6 +875,36 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
             contains(res, "OU=12,OU=SEARCH_NON_UNIQUE01,DC=SAMBA,DC=ORG"))
 
     #
+    # Test index key truncation for base64 encoded values
+    #
+    def test_index_truncated_base64_encoded_keys(self):
+        value = b"aaaaaaaaaaaaaaaaaaaa\x02"
+        # base64 encodes to "YWFhYWFhYWFhYWFhYWFhYWFhYWEC"
+
+        # One less than max key length
+        self.l.add({"dn": "OU=01,OU=BASE64,DC=SAMBA,DC=ORG",
+                    "base64____lt": value,
+                    "objectUUID": b"0123456789abcde0"})
+        self.checkGuids(
+            "@INDEX:BASE64____LT::YWFhYWFhYWFhYWFhYWFhYWFhYWEC",
+            b"0123456789abcde0")
+
+        # Equal max key length
+        self.l.add({"dn": "OU=02,OU=BASE64,DC=SAMBA,DC=ORG",
+                    "base64_____eq": value,
+                    "objectUUID": b"0123456789abcde1"})
+        self.checkGuids(
+            "@INDEX:BASE64_____EQ::YWFhYWFhYWFhYWFhYWFhYWFhYWEC",
+            b"0123456789abcde1")
+
+        # One greater than max key length
+        self.l.add({"dn": "OU=03,OU=BASE64,DC=SAMBA,DC=ORG",
+                    "base64______gt": value,
+                    "objectUUID": b"0123456789abcde2"})
+        self.checkGuids(
+            "@INDEX#BASE64______GT##YWFhYWFhYWFhYWFhYWFhYWFhYWE",
+            b"0123456789abcde2")
+    #
     # Test adding to non unique index with identical multivalued index
     # attributes
     #
-- 
2.11.0


From c46936443f377ae3d84b957ad490039edbec3387 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 12:43:25 +1300
Subject: [PATCH 16/54] ldb test: close pipes to stop forked tests failing on
 failure

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index dfc0209260e..23decd5b585 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -1524,6 +1524,7 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
 		struct ldb_message *msg;
 		TALLOC_FREE(ctx->test_ctx->ldb);
 		TALLOC_FREE(ctx->test_ctx->ev);
+		close(pipes[0]);
 		ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
 		if (ctx->test_ctx->ev == NULL) {
 			exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1586,7 +1587,7 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
 		ret = ldb_transaction_commit(ctx->test_ctx->ldb);
 		exit(ret);
 	}
-
+	close(pipes[1]);
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
 
@@ -1760,6 +1761,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
 		struct ldb_dn *dn, *new_dn;
 		TALLOC_FREE(ctx->test_ctx->ldb);
 		TALLOC_FREE(ctx->test_ctx->ev);
+		close(pipes[0]);
 		ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
 		if (ctx->test_ctx->ev == NULL) {
 			exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1826,6 +1828,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
 		struct ldb_message_element *el;
 		TALLOC_FREE(ctx->test_ctx->ldb);
 		TALLOC_FREE(ctx->test_ctx->ev);
+		close(pipes[0]);
 		ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
 		if (ctx->test_ctx->ev == NULL) {
 			exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1901,6 +1904,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
 	 * sending the "GO" as it is blocked at ldb_transaction_start().
 	 */
 
+	close(pipes[1]);
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
 
@@ -2075,6 +2079,7 @@ static int test_ldb_modify_during_whole_search_callback1(struct ldb_request *req
 		struct ldb_message_element *el;
 		TALLOC_FREE(ctx->test_ctx->ldb);
 		TALLOC_FREE(ctx->test_ctx->ev);
+		close(pipes[0]);
 		ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
 		if (ctx->test_ctx->ev == NULL) {
 			exit(LDB_ERR_OPERATIONS_ERROR);
@@ -2137,6 +2142,7 @@ static int test_ldb_modify_during_whole_search_callback1(struct ldb_request *req
 		exit(ret);
 	}
 
+	close(pipes[1]);
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
 
@@ -2349,6 +2355,7 @@ static void test_ldb_modify_before_ldb_wait(void **state)
 		struct ldb_message_element *el;
 		TALLOC_FREE(search_test_ctx->ldb_test_ctx->ldb);
 		TALLOC_FREE(search_test_ctx->ldb_test_ctx->ev);
+		close(pipes[0]);
 		search_test_ctx->ldb_test_ctx->ev = tevent_context_init(search_test_ctx->ldb_test_ctx);
 		if (search_test_ctx->ldb_test_ctx->ev == NULL) {
 			exit(LDB_ERR_OPERATIONS_ERROR);
@@ -2417,6 +2424,7 @@ static void test_ldb_modify_before_ldb_wait(void **state)
 		ret = ldb_transaction_commit(search_test_ctx->ldb_test_ctx->ldb);
 		exit(ret);
 	}
+	close(pipes[1]);
 
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
-- 
2.11.0


From 7d937d6c00be422f944986593dec1f7efc60ca0f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 16:43:54 +1300
Subject: [PATCH 17/54] ldb-samba: require pid match for cached ldb

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb-samba/ldb_wrap.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 53255556e7c..143e128d5d7 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -94,6 +94,8 @@ static struct ldb_wrap {
 		/* the context is what we use to tell if two ldb
 		 * connections are exactly equivalent
 		 */
+		pid_t pid; /* We want to re-open in a new PID due to
+			    * the LMDB backend */
 		const char *url;
 		struct tevent_context *ev;
 		struct loadparm_context *lp_ctx;
@@ -186,10 +188,12 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
 				   struct cli_credentials *credentials,
 				   unsigned int flags)
 {
+	pid_t pid = getpid();
 	struct ldb_wrap *w;
 	/* see if we can re-use an existing ldb */
 	for (w=ldb_wrap_list; w; w=w->next) {
-		if (w->context.ev == ev &&
+		if (w->context.pid == pid &&
+		    w->context.ev == ev &&
 		    w->context.lp_ctx == lp_ctx &&
 		    w->context.session_info == session_info &&
 		    w->context.credentials == credentials &&
@@ -249,6 +253,7 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 		return false;
 	}
 
+	c.pid          = getpid();
 	c.url          = url;
 	c.ev           = ev;
 	c.lp_ctx       = lp_ctx;
-- 
2.11.0


From 77f6a4fe28516e1a2da15669675028ea2284c90f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:20:35 +1300
Subject: [PATCH 18/54] ldb tests: ldb_mod_op_test use correct ldb to create dn

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 23decd5b585..6582f37f7d2 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3088,7 +3088,7 @@ static void test_read_only(void **state)
 		msg = ldb_msg_new(tmp_ctx);
 		assert_non_null(msg);
 
-		msg->dn = ldb_dn_new_fmt(msg, ro_ldb, "dc=test");
+		msg->dn = ldb_dn_new_fmt(msg, rw_ldb, "dc=test");
 		assert_non_null(msg->dn);
 
 		ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
-- 
2.11.0


From 416ca58aa19a7b43d56349802fbeb474c8bf459a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:33:32 +1300
Subject: [PATCH 19/54] ldb_tdb: ltdb_tdb_parse_record map tdb error codes

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 423b5a1f687..bfd3770c320 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1882,9 +1882,14 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
 		.dptr = ldb_key.data,
 		.dsize = ldb_key.length
 	};
+	int ret;
 
-	return tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
-				&kv_ctx);
+	ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
+			       &kv_ctx);
+	if (ret == 0) {
+		return LDB_SUCCESS;
+	}
+	return ltdb_err_map(tdb_error(ltdb->tdb));
 }
 
 static const char * ltdb_tdb_name(struct ltdb_private *ltdb)
-- 
2.11.0


From 5b407937cf0fd3a66993ed1027e1f6b536cbb6d8 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:36:33 +1300
Subject: [PATCH 20/54] ldb_tdb: ltdb_tdb_store require active transaction

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index bfd3770c320..5476e665e9c 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -428,6 +428,10 @@ static int ltdb_tdb_store(struct ltdb_private *ltdb, struct ldb_val ldb_key,
 		.dptr = ldb_data.data,
 		.dsize = ldb_data.length
 	};
+	bool transaction_active = tdb_transaction_active(ltdb->tdb);
+	if (transaction_active == false){
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
 	return tdb_store(ltdb->tdb, key, data, flags);
 }
 
-- 
2.11.0


From 8587be25322b455c71512378ed1009729f18bcbf Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:37:06 +1300
Subject: [PATCH 21/54] ldb_tdb: ltdb_tdb_delete require active transaction

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 5476e665e9c..1d06566aa30 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -674,6 +674,10 @@ static int ltdb_tdb_delete(struct ltdb_private *ltdb, struct ldb_val ldb_key)
 		.dptr = ldb_key.data,
 		.dsize = ldb_key.length
 	};
+	bool transaction_active = tdb_transaction_active(ltdb->tdb);
+	if (transaction_active == false){
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
 	return tdb_delete(ltdb->tdb, tdb_key);
 }
 
-- 
2.11.0


From 5aec69d6ba327ad673cf201c1a8279fa6e14a0ae Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:15:12 +1300
Subject: [PATCH 22/54] ldb tests: add cmocka tests of kv operations

Add tests for the behaviour the ldb layer expects the key value layer to
provide.  This should make it easier to add another KV store

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_kv_ops_test.c | 1396 +++++++++++++++++++++++++++++++++++++++
 lib/ldb/wscript                 |    9 +-
 2 files changed, 1404 insertions(+), 1 deletion(-)
 create mode 100644 lib/ldb/tests/ldb_kv_ops_test.c

diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
new file mode 100644
index 00000000000..4cfb618ea94
--- /dev/null
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -0,0 +1,1396 @@
+/*
+ * Tests exercising the ldb key value operations.
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+
+/*
+ * A KV module is expected to have the following behaviour
+ *
+ * - A transaction must be open to perform any read, write or delete operation
+ * - Writes and Deletes should not be visible until a transaction is commited
+ * - Nested transactions are not permitted
+ * - transactions can be rolled back and commited.
+ * - supports iteration over all records in the database
+ * - supports the update_in_iterate operation allowing entries to be
+ *   re-keyed.
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+#include "ldb_tdb/ldb_tdb.h"
+
+
+#define DEFAULT_BE  "tdb"
+
+#ifndef TEST_BE
+#define TEST_BE DEFAULT_BE
+#endif /* TEST_BE */
+
+#define NUM_RECS 1024
+
+
+struct test_ctx {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+
+	const char *dbfile;
+	const char *lockfile;   /* lockfile is separate */
+
+	const char *dbpath;
+};
+
+static void unlink_old_db(struct test_ctx *test_ctx)
+{
+	int ret;
+
+	errno = 0;
+	ret = unlink(test_ctx->lockfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+
+	errno = 0;
+	ret = unlink(test_ctx->dbfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+}
+
+static int noconn_setup(void **state)
+{
+	struct test_ctx *test_ctx;
+
+	test_ctx = talloc_zero(NULL, struct test_ctx);
+	assert_non_null(test_ctx);
+
+	test_ctx->ev = tevent_context_init(test_ctx);
+	assert_non_null(test_ctx->ev);
+
+	test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+	assert_non_null(test_ctx->ldb);
+
+	test_ctx->dbfile = talloc_strdup(test_ctx, "kvopstest.ldb");
+	assert_non_null(test_ctx->dbfile);
+
+	test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+					     test_ctx->dbfile);
+	assert_non_null(test_ctx->lockfile);
+
+	test_ctx->dbpath = talloc_asprintf(test_ctx,
+			TEST_BE"://%s", test_ctx->dbfile);
+	assert_non_null(test_ctx->dbpath);
+
+	unlink_old_db(test_ctx);
+	*state = test_ctx;
+	return 0;
+}
+
+static int noconn_teardown(void **state)
+{
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+
+	unlink_old_db(test_ctx);
+	talloc_free(test_ctx);
+	return 0;
+}
+
+static int setup(void **state)
+{
+	struct test_ctx *test_ctx;
+	int ret;
+	struct ldb_ldif *ldif;
+	const char *index_ldif =		\
+		"dn: @INDEXLIST\n"
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+		"\n";
+
+	noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+		ret = ldb_add(test_ctx->ldb, ldif->msg);
+		assert_int_equal(ret, LDB_SUCCESS);
+	}
+	*state = test_ctx;
+	return 0;
+}
+
+static int teardown(void **state)
+{
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	noconn_teardown((void **) &test_ctx);
+	return 0;
+}
+
+static struct ltdb_private *get_ltdb(struct ldb_context *ldb)
+{
+	void *data = NULL;
+	struct ltdb_private *ltdb = NULL;
+
+	data = ldb_module_get_private(ldb->modules);
+	assert_non_null(data);
+
+	ltdb = talloc_get_type(data, struct ltdb_private);
+	assert_non_null(ltdb);
+
+	return ltdb;
+}
+
+static int parse(struct ldb_val key,
+		 struct ldb_val data,
+		 void *private_data)
+{
+	struct ldb_val* read = private_data;
+
+	/* Yes, we essentially leak this.  That is OK */
+	read->data = talloc_size(talloc_autofree_context(),
+				 data.length);
+	assert_non_null(read->data);
+	
+	memcpy(read->data, data.data, data.length);
+	read->length = data.length;
+	return LDB_SUCCESS;
+}
+
+/*
+ * Test that data can be written to the kv store and be read back.
+ */
+static void test_add_get(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	talloc_free(tmp_ctx);
+}
+
+/*
+ * Test that data can be deleted from the kv store
+ */
+static void test_delete(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now delete it.
+	 */
+	ret = ltdb->kv_ops->delete(ltdb, key);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now try to read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+ * Check that writes are correctly rolled back when a transaction
+ * is rolled back.
+ */
+static void test_transaction_abort_write(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+
+
+	/*
+	 * Now abort the transaction
+	 */
+	ret = ltdb->kv_ops->abort_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back, should not be there
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+ * Check that deletes are correctly rolled back when a transaction is
+ * aborted.
+ */
+static void test_transaction_abort_delete(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now delete it.
+	 */
+	ret = ltdb->kv_ops->delete(ltdb, key);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+
+	/*
+	 * Abort the transaction
+	 */
+	ret = ltdb->kv_ops->abort_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now try to read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+ * Test that writes outside a transaction fail
+ */
+static void test_write_outside_transaction(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Attempt to write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+ * Test data can not be deleted outside a transaction
+ */
+static void test_delete_outside_transaction(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now attempt to delete a record
+	 */
+	ret = ltdb->kv_ops->delete(ltdb, key);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	/*
+	 * And now read it back
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, 0);
+	assert_int_equal(sizeof(value), read.length);
+	assert_memory_equal(value, read.data, sizeof(value));
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+static int traverse_fn(struct ltdb_private *ltdb,
+		       struct ldb_val key,
+		       struct ldb_val data,
+		       void *ctx) {
+
+	int *visits = ctx;
+	int i;
+
+	if (strncmp("key ", (char *) key.data, 4) == 0) {
+		i = strtol((char *) &key.data[4], NULL, 10);
+		visits[i]++;
+	}
+	return LDB_SUCCESS;
+}
+
+
+/*
+ * Test that iterate visits all the records.
+ */
+static void test_iterate(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	int i;
+	int num_recs = 1024;
+	int visits[num_recs];
+
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the records
+	 */
+	for (i = 0; i < num_recs; i++) {
+		struct ldb_val key;
+		struct ldb_val rec;
+		int flags = 0;
+
+		visits[i] = 0;
+		key.data   = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+		key.length = strlen((char *)key.data) + 1;
+
+		rec.data = (uint8_t *) talloc_asprintf(tmp_ctx,
+						       "data for record (%04d)",
+						       i);
+		rec.length = strlen((char *)rec.data) + 1;
+
+		ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+		assert_int_equal(ret, 0);
+
+		TALLOC_FREE(key.data);
+		TALLOC_FREE(rec.data);
+	}
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now iterate over the kv store and ensure that all the
+	 * records are visited.
+	 */
+	ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+	ret = ltdb->kv_ops->iterate(ltdb, traverse_fn, visits);
+	for (i = 0; i <num_recs; i++) {
+		assert_int_equal(1, visits[i]);
+	}
+	ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+	assert_int_equal(ret, 0);
+
+	TALLOC_FREE(tmp_ctx);
+}
+
+/*
+ * Ensure that writes are not visible until the transaction has been
+ * committed.
+ */
+static void test_write_transaction_isolation(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	struct ldb_val key;
+	struct ldb_val val;
+
+	const char *KEY1 = "KEY01";
+	const char *VAL1 = "VALUE01";
+
+	const char *KEY2 = "KEY02";
+	const char *VAL2 = "VALUE02";
+
+	/*
+	 * Pipes etc to co-ordinate the processes
+	 */
+	int to_child[2];
+	int to_parent[2];
+	char buf[2];
+	pid_t pid, w_pid;
+	int wstatus;
+
+	TALLOC_CTX *tmp_ctx;
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+
+	/*
+	 * Add a record to the database
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+	key.length = strlen(KEY1) + 1;
+
+	val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL1);
+	val.length = strlen(VAL1) + 1;
+
+	ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+	assert_int_equal(ret, 0);
+
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+
+	ret = pipe(to_child);
+	assert_int_equal(ret, 0);
+	ret = pipe(to_parent);
+	assert_int_equal(ret, 0);
+	/*
+	 * Now fork a new process
+	 */
+
+	pid = fork();
+	if (pid == 0) {
+
+		struct ldb_context *ldb = NULL;
+		close(to_child[1]);
+		close(to_parent[0]);
+
+		/*
+		 * Wait for the transaction to start
+		 */
+		ret = read(to_child[0], buf, 2);
+		if (ret != 2) {
+			print_error(__location__": read returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		ldb = ldb_init(test_ctx, test_ctx->ev);
+		ret = ldb_connect(ldb, test_ctx->dbpath, 0, NULL);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": ldb_connect returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ltdb = get_ltdb(ldb);
+
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": lock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		/*
+		 * Check that KEY1 is there
+		 */
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+		key.length = strlen(KEY1) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL1) + 1) != val.length) {
+			print_error(__location__": KEY1 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL1) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+			print_error(__location__": KEY1 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL1,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		/*
+		 * Check that KEY2 is not there
+		 */
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+		key.length = strlen(KEY2 + 1);
+
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": lock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_ERR_NO_SUCH_OBJECT) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		/*
+		 * Signal the other process to commit the transaction
+		 */
+		ret = write(to_parent[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__": write returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Wait for the transaction to be commited
+		 */
+		ret = read(to_child[0], buf, 2);
+		if (ret != 2) {
+			print_error(__location__": read returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Check that KEY1 is there
+		 */
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+		key.length = strlen(KEY1) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL1) + 1) != val.length) {
+			print_error(__location__": KEY1 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL1) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+			print_error(__location__": KEY1 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL1,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+
+		/*
+		 * Check that KEY2 is there
+		 */
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+		key.length = strlen(KEY2) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL2) + 1) != val.length) {
+			print_error(__location__": KEY2 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL2) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL2, val.data, strlen(VAL2)) != 0) {
+			print_error(__location__": KEY2 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL2,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		exit(0);
+	}
+	close(to_child[0]);
+	close(to_parent[1]);
+
+	/*
+	 * Begin a transaction and add a record to the database
+	 * but leave the transaction open
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+	key.length = strlen(KEY2) + 1;
+
+	val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL2);
+	val.length = strlen(VAL2) + 1;
+
+	ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Signal the child process
+	 */
+	ret = write(to_child[1], "GO", 2);
+	assert_int_equal(2, ret);
+
+	/*
+	 * Wait for the child process to check the DB state while the
+	 * transaction is active
+	 */
+	ret = read(to_parent[0], buf, 2);
+	assert_int_equal(2, ret);
+
+	/*
+	 * commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(0, ret);
+
+	/*
+	 * Signal the child process
+	 */
+	ret = write(to_child[1], "GO", 2);
+	assert_int_equal(2, ret);
+
+	w_pid = waitpid(pid, &wstatus, 0);
+	assert_int_equal(pid, w_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+
+	TALLOC_FREE(tmp_ctx);
+}
+
+/*
+ * Ensure that deletes are not visible until the transaction has been
+ * committed.
+ */
+static void test_delete_transaction_isolation(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	struct ldb_val key;
+	struct ldb_val val;
+
+	const char *KEY1 = "KEY01";
+	const char *VAL1 = "VALUE01";
+
+	const char *KEY2 = "KEY02";
+	const char *VAL2 = "VALUE02";
+
+	/*
+	 * Pipes etc to co-ordinate the processes
+	 */
+	int to_child[2];
+	int to_parent[2];
+	char buf[2];
+	pid_t pid, w_pid;
+	int wstatus;
+
+	TALLOC_CTX *tmp_ctx;
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+
+	/*
+	 * Add records to the database
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+	key.length = strlen(KEY1) + 1;
+
+	val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL1);
+	val.length = strlen(VAL1) + 1;
+
+	ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+	assert_int_equal(ret, 0);
+
+	key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+	key.length = strlen(KEY2) + 1;
+
+	val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL2);
+	val.length = strlen(VAL2) + 1;
+
+	ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+	assert_int_equal(ret, 0);
+
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+
+	ret = pipe(to_child);
+	assert_int_equal(ret, 0);
+	ret = pipe(to_parent);
+	assert_int_equal(ret, 0);
+	/*
+	 * Now fork a new process
+	 */
+
+	pid = fork();
+	if (pid == 0) {
+
+		struct ldb_context *ldb = NULL;
+		close(to_child[1]);
+		close(to_parent[0]);
+
+		/*
+		 * Wait for the transaction to be started
+		 */
+		ret = read(to_child[0], buf, 2);
+		if (ret != 2) {
+			print_error(__location__": read returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		ldb = ldb_init(test_ctx, test_ctx->ev);
+		ret = ldb_connect(ldb, test_ctx->dbpath, 0, NULL);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": ldb_connect returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ltdb = get_ltdb(ldb);
+
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": lock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		/*
+		 * Check that KEY1 is there
+		 */
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+		key.length = strlen(KEY1) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL1) + 1) != val.length) {
+			print_error(__location__": KEY1 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL1) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+			print_error(__location__": KEY1 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL1,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Check that KEY2 is there
+		 */
+
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+		key.length = strlen(KEY2) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL2) + 1) != val.length) {
+			print_error(__location__": KEY2 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL2) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL2, val.data, strlen(VAL2)) != 0) {
+			print_error(__location__": KEY2 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL2,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		/*
+		 * Signal the other process to commit the transaction
+		 */
+		ret = write(to_parent[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__": write returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Wait for the transaction to be commited
+		 */
+		ret = read(to_child[0], buf, 2);
+		if (ret != 2) {
+			print_error(__location__": read returned (%d)\n",
+				    ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Check that KEY1 is there
+		 */
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+		key.length = strlen(KEY1) + 1;
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		if ((strlen(VAL1) + 1) != val.length) {
+			print_error(__location__": KEY1 value lengths different"
+				    ", expected (%d) actual(%d)\n",
+				    (int)(strlen(VAL1) + 1), (int)val.length);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+			print_error(__location__": KEY1 values different, "
+				    "expected (%s) actual(%s)\n",
+				    VAL1,
+				    val.data);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		/*
+		 * Check that KEY2 is not there
+		 */
+		key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+		key.length = strlen(KEY2 + 1);
+
+		ret = ltdb->kv_ops->lock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": lock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+		if (ret != LDB_ERR_NO_SUCH_OBJECT) {
+			print_error(__location__": fetch_and_parse returned "
+				    "(%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		ret = ltdb->kv_ops->unlock_read(ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			print_error(__location__": unlock_read returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+
+		exit(0);
+	}
+	close(to_child[0]);
+	close(to_parent[1]);
+
+	/*
+	 * Begin a transaction and delete a record from the database
+	 * but leave the transaction open
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+	key.length = strlen(KEY2) + 1;
+
+	ret = ltdb->kv_ops->delete(ltdb, key);
+	assert_int_equal(ret, 0);
+	/*
+	 * Signal the child process
+	 */
+	ret = write(to_child[1], "GO", 2);
+	assert_int_equal(2, ret);
+
+	/*
+	 * Wait for the child process to check the DB state while the
+	 * transaction is active
+	 */
+	ret = read(to_parent[0], buf, 2);
+	assert_int_equal(2, ret);
+
+	/*
+	 * commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(0, ret);
+
+	/*
+	 * Signal the child process
+	 */
+	ret = write(to_child[1], "GO", 2);
+	assert_int_equal(2, ret);
+
+	w_pid = waitpid(pid, &wstatus, 0);
+	assert_int_equal(pid, w_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+
+	TALLOC_FREE(tmp_ctx);
+}
+
+
+int main(int argc, const char **argv)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(
+			test_add_get,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_delete,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_transaction_abort_write,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_transaction_abort_delete,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_write_outside_transaction,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_delete_outside_transaction,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_iterate,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_write_transaction_isolation,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_delete_transaction_isolation,
+			setup,
+			teardown),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index f959d59b28f..81fcb55e79b 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -359,6 +359,12 @@ def build(bld):
                          deps='cmocka ldb',
                          install=False)
 
+        bld.SAMBA_BINARY('ldb_tdb_kv_ops_test',
+                         source='tests/ldb_kv_ops_test.c',
+                         cflags='-DTEST_BE=\"tdb\"',
+                         deps='cmocka ldb',
+                         install=False)
+
         bld.SAMBA_BINARY('ldb_msg_test',
                          source='tests/ldb_msg.c',
                          deps='cmocka ldb',
@@ -393,7 +399,8 @@ def test(ctx):
     cmocka_ret = 0
     for test_exe in ['ldb_tdb_mod_op_test',
                      'ldb_tdb_guid_mod_op_test',
-                     'ldb_msg_test']:
+                     'ldb_msg_test',
+                     'ldb_tdb_kv_ops_test']:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
             cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
 
-- 
2.11.0


From 718d4e6a1341aa6bc2322e1455939254e0a7e2c4 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 4 Apr 2018 16:17:51 +1200
Subject: [PATCH 23/54] python: Add wrapper of mdb_copy that we can call from
 python

This is like the use of tdbbackup for tdb files.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/mdb_util.py | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 python/samba/mdb_util.py

diff --git a/python/samba/mdb_util.py b/python/samba/mdb_util.py
new file mode 100644
index 00000000000..4fc6c3e0258
--- /dev/null
+++ b/python/samba/mdb_util.py
@@ -0,0 +1,40 @@
+# Unix SMB/CIFS implementation.
+# mdb util helpers
+#
+# Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import samba
+import subprocess
+import os
+
+
+def mdb_copy(file1, file2):
+    """Copy mdb file using mdb_copy utility and rename it
+    """
+    # Find the location of the mdb_copy tool
+    dirs = os.getenv('PATH').split(os.pathsep)
+    for d in dirs:
+        toolpath = os.path.join(d, "mdb_copy")
+        if os.path.exists(toolpath):
+            break
+
+    mdb_copy_cmd = [toolpath, "-n", file1, "%s.copy.mdb" % file1]
+    status = subprocess.call(mdb_copy_cmd, close_fds=True, shell=False)
+
+    if status == 0:
+        os.rename("%s.copy.mdb" % file1, file2)
+    else:
+        raise Exception("Error copying %d  %s" % (status, file1))
-- 
2.11.0


From dd92502e0a40e9edabef87882e193f0b1b5f2ac1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 14:38:19 +1300
Subject: [PATCH 24/54] provision: allow provisioning of a different database
 backend

This sets the backendStore field in @PARTITION, depending on which
argument you set in the provision.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/netcmd/domain.py           | 14 +++++++++---
 python/samba/provision/__init__.py      | 38 +++++++++++++++++++++++----------
 python/samba/provision/sambadns.py      | 35 ++++++++++++++++++++++--------
 python/samba/samdb.py                   |  3 +++
 source4/setup/provision_partitions.ldif |  1 +
 5 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/python/samba/netcmd/domain.py b/python/samba/netcmd/domain.py
index 963d8822e6c..d2dd06a3d48 100644
--- a/python/samba/netcmd/domain.py
+++ b/python/samba/netcmd/domain.py
@@ -43,7 +43,7 @@ from samba.net import Net, LIBNET_JOIN_AUTOMATIC
 import samba.ntacls
 from samba.join import join_RODC, join_DC, join_subdomain
 from samba.auth import system_session
-from samba.samdb import SamDB
+from samba.samdb import SamDB, get_default_backend_store
 from samba.ndr import ndr_unpack, ndr_pack, ndr_print
 from samba.dcerpc import drsuapi
 from samba.dcerpc import drsblobs
@@ -258,6 +258,10 @@ class cmd_domain_provision(Command):
          Option("--plaintext-secrets", action="store_true",
                 help="Store secret/sensitive values as plain text on disk" +
                      "(default is to encrypt secret/ensitive values)"),
+         Option("--backend-store", type="choice", metavar="BACKENDSTORE",
+                choices=["tdb", "mdb"],
+                help="Specify the database backend to be used "
+                     "(default is %s)" % get_default_backend_store()),
         ]
 
     openldap_options = [
@@ -328,7 +332,8 @@ class cmd_domain_provision(Command):
             ldap_backend_forced_uri=None,
             ldap_dryrun_mode=None,
             base_schema=None,
-            plaintext_secrets=False):
+            plaintext_secrets=False,
+            backend_store=None):
 
         self.logger = self.get_logger("provision")
         if quiet:
@@ -476,6 +481,8 @@ class cmd_domain_provision(Command):
             domain_sid = security.dom_sid(domain_sid)
 
         session = system_session()
+        if backend_store is None:
+            backend_store = get_default_backend_store()
         try:
             result = provision(self.logger,
                   session, smbconf=smbconf, targetdir=targetdir,
@@ -498,7 +505,8 @@ class cmd_domain_provision(Command):
                   ldap_backend_forced_uri=ldap_backend_forced_uri,
                   nosync=ldap_backend_nosync, ldap_dryrun_mode=ldap_dryrun_mode,
                   base_schema=base_schema,
-                  plaintext_secrets=plaintext_secrets)
+                  plaintext_secrets=plaintext_secrets,
+                  backend_store=backend_store)
 
         except ProvisioningError as e:
             raise CommandError("Provision failed", e)
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index f36f277742e..aaf839cc3f8 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -123,6 +123,7 @@ from samba.schema import Schema
 from samba.samdb import SamDB
 from samba.dbchecker import dbcheck
 from samba.provision.kerberos import create_kdc_conf
+from samba.samdb import get_default_backend_store
 
 DEFAULT_POLICY_GUID = "31B2F340-016D-11D2-945F-00C04FB984F9"
 DEFAULT_DC_POLICY_GUID = "6AC1786C-016F-11D2-945F-00C04FB984F9"
@@ -814,7 +815,8 @@ def setup_name_mappings(idmap, sid, root_uid, nobody_uid,
 
 def setup_samdb_partitions(samdb_path, logger, lp, session_info,
                            provision_backend, names, serverrole,
-                           erase=False, plaintext_secrets=False):
+                           erase=False, plaintext_secrets=False,
+                           backend_store=None):
     """Setup the partitions for the SAM database.
 
     Alternatively, provision() may call this, and then populate the database.
@@ -847,11 +849,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
     if not plaintext_secrets:
         required_features = "requiredFeatures: encryptedSecrets"
 
+    if backend_store is None:
+        backend_store = get_default_backend_store()
+    backend_store_line = "backendStore: %s" % backend_store
+
     samdb.transaction_start()
     try:
         logger.info("Setting up sam.ldb partitions and settings")
         setup_add_ldif(samdb, setup_path("provision_partitions.ldif"), {
-                "LDAP_BACKEND_LINE": ldap_backend_line
+                "LDAP_BACKEND_LINE": ldap_backend_line,
+                "BACKEND_STORE": backend_store_line
         })
 
 
@@ -1245,7 +1252,7 @@ def create_default_gpo(sysvolpath, dnsdomain, policyguid, policyguid_dc):
 
 def setup_samdb(path, session_info, provision_backend, lp, names,
         logger, fill, serverrole, schema, am_rodc=False,
-        plaintext_secrets=False):
+        plaintext_secrets=False, backend_store=None):
     """Setup a complete SAM Database.
 
     :note: This will wipe the main SAM database file!
@@ -1254,7 +1261,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
     # Also wipes the database
     setup_samdb_partitions(path, logger=logger, lp=lp,
         provision_backend=provision_backend, session_info=session_info,
-        names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets)
+        names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets,
+        backend_store=backend_store)
 
     # Load the database, but don's load the global schema and don't connect
     # quite yet
@@ -1293,7 +1301,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
 def fill_samdb(samdb, lp, names, logger, policyguid,
         policyguid_dc, fill, adminpass, krbtgtpass, machinepass, dns_backend,
         dnspass, invocationid, ntdsguid, serverrole, am_rodc=False,
-        dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None):
+        dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None,
+        backend_store=None):
 
     if next_rid is None:
         next_rid = 1000
@@ -1831,7 +1840,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
                    invocationid=None, machinepass=None, ntdsguid=None,
                    dns_backend=None, dnspass=None,
                    serverrole=None, dom_for_fun_level=None,
-                   am_rodc=False, lp=None, use_ntvfs=False, skip_sysvolacl=False):
+                   am_rodc=False, lp=None, use_ntvfs=False,
+                   skip_sysvolacl=False, backend_store=None):
     # create/adapt the group policy GUIDs
     # Default GUID for default policy are described at
     # "How Core Group Policy Works"
@@ -1863,7 +1873,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
                        dns_backend=dns_backend, dnspass=dnspass,
                        ntdsguid=ntdsguid, serverrole=serverrole,
                        dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
-                       next_rid=next_rid, dc_rid=dc_rid)
+                       next_rid=next_rid, dc_rid=dc_rid,
+                       backend_store=backend_store)
 
         # Set up group policies (domain policy and domain controller
         # policy)
@@ -1913,7 +1924,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
         setup_ad_dns(samdb, secrets_ldb, names, paths, lp, logger,
                      hostip=hostip, hostip6=hostip6, dns_backend=dns_backend,
                      dnspass=dnspass, os_level=dom_for_fun_level,
-                     targetdir=targetdir, fill_level=samdb_fill)
+                     targetdir=targetdir, fill_level=samdb_fill,
+                     backend_store=backend_store)
 
         domainguid = samdb.searchone(basedn=samdb.get_default_basedn(),
                                      attribute="objectGUID")
@@ -2033,7 +2045,7 @@ def provision(logger, session_info, smbconf=None,
         use_rfc2307=False, maxuid=None, maxgid=None, skip_sysvolacl=True,
         ldap_backend_forced_uri=None, nosync=False, ldap_dryrun_mode=False,
         ldap_backend_extra_port=None, base_schema=None,
-        plaintext_secrets=False):
+        plaintext_secrets=False, backend_store=None):
     """Provision samba4
 
     :note: caution, this wipes all existing data!
@@ -2050,6 +2062,8 @@ def provision(logger, session_info, smbconf=None,
 
     if backend_type is None:
         backend_type = "ldb"
+    if backend_store is None:
+        backend_store = get_default_backend_store()
 
     if domainsid is None:
         domainsid = security.random_sid()
@@ -2233,7 +2247,8 @@ def provision(logger, session_info, smbconf=None,
                             provision_backend, lp, names, logger=logger,
                             serverrole=serverrole,
                             schema=schema, fill=samdb_fill, am_rodc=am_rodc,
-                            plaintext_secrets=plaintext_secrets)
+                            plaintext_secrets=plaintext_secrets,
+                            backend_store=backend_store)
 
         if serverrole == "active directory domain controller":
             if paths.netlogon is None:
@@ -2264,7 +2279,8 @@ def provision(logger, session_info, smbconf=None,
                     dnspass=dnspass, serverrole=serverrole,
                     dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
                     lp=lp, use_ntvfs=use_ntvfs,
-                           skip_sysvolacl=skip_sysvolacl)
+                    skip_sysvolacl=skip_sysvolacl,
+                    backend_store=backend_store)
 
         if not is_heimdal_built():
             create_kdc_conf(paths.kdcconf, realm, domain, os.path.dirname(lp.get("log file")))
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index 4cc15b06700..ce1b7692ff9 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -29,6 +29,7 @@ from base64 import b64encode
 import subprocess
 import samba
 from samba.tdb_util import tdb_copy
+from samba.mdb_util import mdb_copy
 from samba.ndr import ndr_pack, ndr_unpack
 from samba import setup_file
 from samba.dcerpc import dnsp, misc, security
@@ -58,6 +59,7 @@ from samba.provision.common import (
     FILL_DRS,
     )
 
+from samba.samdb import get_default_backend_store
 
 def get_domainguid(samdb, domaindn):
     res = samdb.search(base=domaindn, scope=ldb.SCOPE_BASE, attrs=["objectGUID"])
@@ -787,12 +789,19 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
 
     # Find the partitions and corresponding filenames
     partfile = {}
-    res = samdb.search(base="@PARTITION", scope=ldb.SCOPE_BASE, attrs=["partition"])
+    res = samdb.search(base="@PARTITION",
+                       scope=ldb.SCOPE_BASE,
+                       attrs=["partition", "backendStore"])
     for tmp in res[0]["partition"]:
         (nc, fname) = tmp.split(':')
         partfile[nc.upper()] = fname
 
+    backend_store = get_default_backend_store()
+    if "backendStore" in res[0]:
+        backend_store = res[0]["backendStore"][0]
+
     # Create empty domain partition
+
     domaindn = names.domaindn.upper()
     domainpart_file = os.path.join(dns_dir, partfile[domaindn])
     try:
@@ -800,7 +809,8 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
         file(domainpart_file, 'w').close()
 
         # Fill the basedn and @OPTION records in domain partition
-        dom_ldb = samba.Ldb(domainpart_file)
+        dom_url = "%s://%s" % (backend_store, domainpart_file)
+        dom_ldb = samba.Ldb(dom_url)
         domainguid_line = "objectGUID: %s\n-" % domainguid
         descr = b64encode(get_domain_descriptor(domainsid))
         setup_add_ldif(dom_ldb, setup_path("provision_basedn.ldif"), {
@@ -859,8 +869,12 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
                  os.path.join(dns_dir, "sam.ldb"))
         for nc in partfile:
             pfile = partfile[nc]
-            tdb_copy(os.path.join(private_dir, pfile),
-                     os.path.join(dns_dir, pfile))
+            if backend_store == "mdb":
+                mdb_copy(os.path.join(private_dir, pfile),
+                        os.path.join(dns_dir, pfile))
+            else:
+                tdb_copy(os.path.join(private_dir, pfile),
+                        os.path.join(dns_dir, pfile))
     except:
         logger.error(
             "Failed to setup database for BIND, AD based DNS cannot be used")
@@ -875,7 +889,7 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
                     os.chown(dpath, -1, paths.bind_gid)
                     os.chmod(dpath, 0o770)
                 for f in files:
-                    if f.endswith('.ldb') or f.endswith('.tdb'):
+                    if f.endswith(('.ldb', '.tdb', 'ldb-lock')):
                         fpath = os.path.join(dirname, f)
                         os.chown(fpath, -1, paths.bind_gid)
                         os.chmod(fpath, 0o660)
@@ -1069,7 +1083,7 @@ def fill_dns_data_partitions(samdb, domainsid, site, domaindn, forestdn,
 
 def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
         dns_backend, os_level, dnspass=None, hostip=None, hostip6=None,
-        targetdir=None, fill_level=FILL_FULL):
+        targetdir=None, fill_level=FILL_FULL, backend_store=None):
     """Provision DNS information (assuming GC role)
 
     :param samdb: LDB object connected to sam.ldb file
@@ -1164,12 +1178,14 @@ def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
     if dns_backend.startswith("BIND9_"):
         setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
                         dns_backend, os_level, site=site, dnspass=dnspass, hostip=hostip,
-                        hostip6=hostip6, targetdir=targetdir)
+                        hostip6=hostip6, targetdir=targetdir,
+                        backend_store=backend_store)
 
 
 def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
         dns_backend, os_level, site=None, dnspass=None, hostip=None,
-        hostip6=None, targetdir=None, key_version_number=None):
+        hostip6=None, targetdir=None, key_version_number=None,
+        backend_store=None):
     """Provision DNS information (assuming BIND9 backend in DC role)
 
     :param samdb: LDB object connected to sam.ldb file
@@ -1216,7 +1232,8 @@ def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
                          ntdsguid=names.ntdsguid)
 
     if dns_backend == "BIND9_DLZ" and os_level >= DS_DOMAIN_FUNCTION_2003:
-        create_samdb_copy(samdb, logger, paths, names, names.domainsid, domainguid)
+        create_samdb_copy(samdb, logger, paths,
+                          names, names.domainsid, domainguid)
 
     create_named_conf(paths, realm=names.realm,
                       dnsdomain=names.dnsdomain, dns_backend=dns_backend,
diff --git a/python/samba/samdb.py b/python/samba/samdb.py
index 348bd212256..d7faa236f88 100644
--- a/python/samba/samdb.py
+++ b/python/samba/samdb.py
@@ -37,6 +37,9 @@ from samba.dcerpc import security
 __docformat__ = "restructuredText"
 
 
+def get_default_backend_store():
+    return "tdb"
+
 class SamDB(samba.Ldb):
     """The SAM database."""
 
diff --git a/source4/setup/provision_partitions.ldif b/source4/setup/provision_partitions.ldif
index 728f32739d4..4cd57943228 100644
--- a/source4/setup/provision_partitions.ldif
+++ b/source4/setup/provision_partitions.ldif
@@ -2,5 +2,6 @@ dn: @PARTITION
 replicateEntries: @ATTRIBUTES
 replicateEntries: @INDEXLIST
 replicateEntries: @OPTIONS
+${BACKEND_STORE}
 ${LDAP_BACKEND_LINE}
 
-- 
2.11.0


From a8f3a39dc1a40e5c6124f9095aebb140088ea2f0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 10:58:11 +1300
Subject: [PATCH 25/54] provision: Set @INDEXLIST first when building dummy
 sam.ldb

The new LMDB backed will not allow normal records to be added before the @INDEXLIST
as this is what forces the GUID index mode.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/provision/sambadns.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index ce1b7692ff9..7a85546c53e 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -811,6 +811,11 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
         # Fill the basedn and @OPTION records in domain partition
         dom_url = "%s://%s" % (backend_store, domainpart_file)
         dom_ldb = samba.Ldb(dom_url)
+
+        # We need the dummy main-domain DB to have the correct @INDEXLIST
+        index_res = samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE)
+        dom_ldb.add(index_res[0])
+
         domainguid_line = "objectGUID: %s\n-" % domainguid
         descr = b64encode(get_domain_descriptor(domainsid))
         setup_add_ldif(dom_ldb, setup_path("provision_basedn.ldif"), {
@@ -821,9 +826,6 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
         setup_add_ldif(dom_ldb,
             setup_path("provision_basedn_options.ldif"), None)
 
-        # We need the dummy main-domain DB to have the correct @INDEXLIST
-        index_res = samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE)
-        dom_ldb.add(index_res[0])
     except:
         logger.error(
             "Failed to setup database for BIND, AD based DNS cannot be used")
-- 
2.11.0


From 3d6757fab4ccb1ddaea20207198c59dd13fad5fc Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Feb 2018 11:10:34 +1300
Subject: [PATCH 26/54] dsdb: add lmdbLevelOne as a required feature.

---
 python/samba/provision/__init__.py          | 12 +++++++++++-
 source4/dsdb/samdb/ldb_modules/samba_dsdb.c |  5 ++++-
 source4/dsdb/samdb/samdb.h                  |  9 +++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index aaf839cc3f8..dcdb2b7bd20 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -845,7 +845,7 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
     if provision_backend.type != "ldb":
         ldap_backend_line = "ldapBackend: %s" % provision_backend.ldap_uri
 
-    required_features = "# No required features"
+    required_features = None
     if not plaintext_secrets:
         required_features = "requiredFeatures: encryptedSecrets"
 
@@ -853,6 +853,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
         backend_store = get_default_backend_store()
     backend_store_line = "backendStore: %s" % backend_store
 
+    if backend_store == "mdb":
+        if required_features is not None:
+            required_features += "\n"
+        else:
+            required_features = ""
+        required_features += "requiredFeatures: lmdbLevelOne"
+
+    if required_features is None:
+        required_features = "# No required features"
+
     samdb.transaction_start()
     try:
         logger.info("Setting up sam.ldb partitions and settings")
diff --git a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
index e0acb4e371a..2605c1e511e 100644
--- a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
+++ b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
@@ -234,8 +234,11 @@ static bool check_required_features(struct ldb_message_element *el)
 		int k;
 		DATA_BLOB esf = data_blob_string_const(
 			SAMBA_ENCRYPTED_SECRETS_FEATURE);
+		DATA_BLOB lmdbl1 = data_blob_string_const(
+			SAMBA_LMDB_LEVEL_ONE_FEATURE);
 		for (k = 0; k < el->num_values; k++) {
-			if (data_blob_cmp(&esf, &el->values[k]) != 0) {
+			if ((data_blob_cmp(&esf, &el->values[k]) != 0) &&
+			    (data_blob_cmp(&lmdbl1, &el->values[k]) != 0)) {
 				return false;
 			}
 		}
diff --git a/source4/dsdb/samdb/samdb.h b/source4/dsdb/samdb/samdb.h
index eb527402806..a095858d572 100644
--- a/source4/dsdb/samdb/samdb.h
+++ b/source4/dsdb/samdb/samdb.h
@@ -342,5 +342,14 @@ struct dsdb_extended_sec_desc_propagation_op {
 
 #define SAMBA_SORTED_LINKS_FEATURE "sortedLinks"
 #define SAMBA_ENCRYPTED_SECRETS_FEATURE "encryptedSecrets"
+/*
+ * lmdb level one feature is an experimental release with basic support
+ * for lmdb database files, instead of tdb.
+ * - Keys are limited to 511 bytes long so GUID indexes are required
+ * - Currently only the:
+ *     partition data files
+ *   are in lmdb format.
+ */
+#define SAMBA_LMDB_LEVEL_ONE_FEATURE "lmdbLevelOne"
 
 #endif /* __SAMDB_H__ */
-- 
2.11.0


From f5a569d77524c69b4261207b5c43262665cc29e3 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 13:14:38 +1300
Subject: [PATCH 27/54] source3: initilize_password_db after a fork.

This is required because we need a new pointer for LDB after the fork,
and with LMDB we can not longer rely on tdb_reopen_all() to do that
for us.

This can not be done in reinit_after_fork() due to the dependency loop
this would create.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source3/rpc_server/lsasd.c       | 3 +++
 source3/smbd/process.c           | 1 +
 source3/smbd/server_exit.c       | 6 +++++-
 source3/winbindd/winbindd.c      | 2 ++
 source3/winbindd/winbindd_dual.c | 2 ++
 5 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/source3/rpc_server/lsasd.c b/source3/rpc_server/lsasd.c
index 08e9fe26a74..9dcf4624aea 100644
--- a/source3/rpc_server/lsasd.c
+++ b/source3/rpc_server/lsasd.c
@@ -22,6 +22,7 @@
 #include "includes.h"
 #include "messages.h"
 #include "ntdomain.h"
+#include "passdb.h"
 
 #include "lib/id_cache.h"
 
@@ -251,6 +252,7 @@ static bool lsasd_child_init(struct tevent_context *ev_ctx,
 		DEBUG(0,("reinit_after_fork() failed\n"));
 		smb_panic("reinit_after_fork() failed");
 	}
+	initialize_password_db(true, ev_ctx);
 
 	lsasd_child_id = child_id;
 	lsasd_reopen_logs(child_id);
@@ -856,6 +858,7 @@ void start_lsasd(struct tevent_context *ev_ctx,
 		DEBUG(0,("reinit_after_fork() failed\n"));
 		smb_panic("reinit_after_fork() failed");
 	}
+	initialize_password_db(true, ev_ctx);
 
 	/* save the parent process id so the children can use it later */
 	parent_id = messaging_server_id(msg_ctx);
diff --git a/source3/smbd/process.c b/source3/smbd/process.c
index 4376f2eff99..df54a44b884 100644
--- a/source3/smbd/process.c
+++ b/source3/smbd/process.c
@@ -3412,6 +3412,7 @@ bool fork_echo_handler(struct smbXsrv_connection *xconn)
 				  nt_errstr(status)));
 			exit(1);
 		}
+		initialize_password_db(true, xconn->ev_ctx);
 		smbd_echo_loop(xconn, listener_pipe[1]);
 		exit(0);
 	}
diff --git a/source3/smbd/server_exit.c b/source3/smbd/server_exit.c
index dbeb247c170..50b43a1ced4 100644
--- a/source3/smbd/server_exit.c
+++ b/source3/smbd/server_exit.c
@@ -44,6 +44,7 @@
 #include "printing.h"
 #include "serverid.h"
 #include "messages.h"
+#include "passdb.h"
 #include "../lib/util/pidfile.h"
 #include "smbprofile.h"
 #include "libcli/auth/netlogon_creds_cli.h"
@@ -261,6 +262,9 @@ NTSTATUS smbd_reinit_after_fork(struct messaging_context *msg_ctx,
 				struct tevent_context *ev_ctx,
 				bool parent_longlived, const char *comment)
 {
+	NTSTATUS ret;
 	am_parent = NULL;
-	return reinit_after_fork(msg_ctx, ev_ctx, parent_longlived, comment);
+	ret = reinit_after_fork(msg_ctx, ev_ctx, parent_longlived, comment);
+	initialize_password_db(true, ev_ctx);
+	return ret;
 }
diff --git a/source3/winbindd/winbindd.c b/source3/winbindd/winbindd.c
index 8821f39a0da..b908d91e206 100644
--- a/source3/winbindd/winbindd.c
+++ b/source3/winbindd/winbindd.c
@@ -44,6 +44,7 @@
 #include "lib/async_req/async_sock.h"
 #include "libsmb/samlogon_cache.h"
 #include "libcli/auth/netlogon_creds_cli.h"
+#include "passdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -1752,6 +1753,7 @@ int main(int argc, const char **argv)
 	if (!NT_STATUS_IS_OK(status)) {
 		exit_daemon("Winbindd reinit_after_fork() failed", map_errno_from_nt_status(status));
 	}
+	initialize_password_db(true, server_event_context());
 
 	/*
 	 * Do not initialize the parent-child-pipe before becoming
diff --git a/source3/winbindd/winbindd_dual.c b/source3/winbindd/winbindd_dual.c
index 5ae5bbd9468..cff2c974dee 100644
--- a/source3/winbindd/winbindd_dual.c
+++ b/source3/winbindd/winbindd_dual.c
@@ -40,6 +40,7 @@
 #include "lib/param/loadparm.h"
 #include "lib/util/sys_rw.h"
 #include "lib/util/sys_rw_data.h"
+#include "passdb.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_WINBIND
@@ -1503,6 +1504,7 @@ NTSTATUS winbindd_reinit_after_fork(const struct winbindd_child *myself,
 		DEBUG(0,("reinit_after_fork() failed\n"));
 		return status;
 	}
+	initialize_password_db(true, server_event_context());
 
 	close_conns_after_fork();
 
-- 
2.11.0


From 1943d41beff2ab0c6095bead7a6044a008b0cc42 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:23:39 +1300
Subject: [PATCH 28/54] ldb: Unwind transaction counter if start_transaction
 fails

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
 lib/ldb/common/ldb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/ldb/common/ldb.c b/lib/ldb/common/ldb.c
index a4d9977d1b4..2249089d087 100644
--- a/lib/ldb/common/ldb.c
+++ b/lib/ldb/common/ldb.c
@@ -379,6 +379,7 @@ int ldb_transaction_start(struct ldb_context *ldb)
 				"ldb transaction start: %s (%d)",
 				ldb_strerror(status),
 				status);
+		ldb->transaction_active--;
 		}
 		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
 			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "start ldb transaction error: %s",
-- 
2.11.0


From 0224a02cd7cf03fe3228728456971fe045dfb9e5 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 22 Mar 2018 12:50:45 +1300
Subject: [PATCH 29/54] upgradeprovision: detect and handle lmdb databases

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/scripting/bin/samba_upgradeprovision | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/source4/scripting/bin/samba_upgradeprovision b/source4/scripting/bin/samba_upgradeprovision
index e0db12e0818..62b7001f565 100755
--- a/source4/scripting/bin/samba_upgradeprovision
+++ b/source4/scripting/bin/samba_upgradeprovision
@@ -36,11 +36,13 @@ sys.path.insert(0, "bin/python")
 import ldb
 import samba
 import samba.getopt as options
+from samba.samdb import get_default_backend_store
 
 from base64 import b64encode
 from samba.credentials import DONT_USE_KERBEROS
 from samba.auth import system_session, admin_session
 from samba import tdb_util
+from samba import mdb_util
 from ldb import (SCOPE_SUBTREE, SCOPE_BASE,
                 FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE,
                 MessageElement, Message, Dn, LdbError)
@@ -1366,7 +1368,7 @@ def update_samdb(ref_samdb, samdb, names, provisionUSNs, schema, prereloadfunc):
         return 0
 
 
-def backup_provision(paths, dir, only_db):
+def backup_provision(samdb, paths, dir, only_db):
     """This function backup the provision files so that a rollback
     is possible
 
@@ -1374,8 +1376,20 @@ def backup_provision(paths, dir, only_db):
     :param dir: Directory where to store the backup
     :param only_db: Skip sysvol for users with big sysvol
     """
+
+    # Currently we default to tdb for the backend store type
+    #
+    backend_store = "tdb"
+    res = samdb.search(base="@PARTITION",
+                       scope=ldb.SCOPE_BASE,
+                       attrs=["backendStore"])
+    if "backendStore" in res[0]:
+        backend_store = res[0]["backendStore"][0]
+
+
     if paths.sysvol and not only_db:
         copytree_with_xattrs(paths.sysvol, os.path.join(dir, "sysvol"))
+
     tdb_util.tdb_copy(paths.samdb, os.path.join(dir, os.path.basename(paths.samdb)))
     tdb_util.tdb_copy(paths.secrets, os.path.join(dir, os.path.basename(paths.secrets)))
     tdb_util.tdb_copy(paths.idmapdb, os.path.join(dir, os.path.basename(paths.idmapdb)))
@@ -1399,8 +1413,12 @@ def backup_provision(paths, dir, only_db):
 
         for ldb_name in os.listdir(samldbdir):
             if not ldb_name.endswith("-lock"):
-                tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
-                                  os.path.join(dir, "sam.ldb.d", ldb_name))
+                if backend_store == "mdb" and ldb_name != "metadata.tdb":
+                    mdb_util.mdb_copy(os.path.join(samldbdir, ldb_name),
+                                      os.path.join(dir, "sam.ldb.d", ldb_name))
+                else:
+                    tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
+                                      os.path.join(dir, "sam.ldb.d", ldb_name))
 
 
 def sync_calculated_attributes(samdb, names):
@@ -1571,7 +1589,7 @@ if __name__ == '__main__':
     ldbs = get_ldbs(paths, creds, session, lp)
     backupdir = tempfile.mkdtemp(dir=paths.private_dir,
                                     prefix="backupprovision")
-    backup_provision(paths, backupdir, opts.db_backup_only)
+    backup_provision(ldbs.sam, paths, backupdir, opts.db_backup_only)
     try:
         ldbs.startTransactions()
 
-- 
2.11.0


From 00a0461f622fbb8fd2cbe7900dbe8e5a6e9d5ad3 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 30/54] ldb_mdb: Implement the lmdb backend for ldb

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 711 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.h |  57 ++++
 lib/ldb/ldb_tdb/ldb_tdb.h |   1 +
 lib/ldb/tools/ldbdump.c   | 126 +++++++-
 lib/ldb/wscript           |  89 +++++-
 5 files changed, 977 insertions(+), 7 deletions(-)
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 00000000000..8bc82715390
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,711 @@
+/*
+   ldb database library using mdb back end
+
+   Copyright (C) Jakub Hrozek 2014
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "lmdb.h"
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX		"mdb://"
+#define MDB_URL_PREFIX_SIZE	(sizeof(MDB_URL_PREFIX)-1)
+
+#define MEGABYTE (1024*1024)
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+	switch (lmdb_err) {
+	case MDB_SUCCESS:
+		return LDB_SUCCESS;
+	case EIO:
+		return LDB_ERR_OPERATIONS_ERROR;
+	case MDB_INCOMPATIBLE:
+	case MDB_CORRUPTED:
+	case MDB_INVALID:
+		return LDB_ERR_UNAVAILABLE;
+	case MDB_BAD_TXN:
+	case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+	case MDB_BAD_DBI:
+#endif
+	case MDB_PANIC:
+	case EINVAL:
+		return LDB_ERR_PROTOCOL_ERROR;
+	case MDB_MAP_FULL:
+	case MDB_DBS_FULL:
+	case MDB_READERS_FULL:
+	case MDB_TLS_FULL:
+	case MDB_TXN_FULL:
+	case EAGAIN:
+		return LDB_ERR_BUSY;
+	case MDB_KEYEXIST:
+		return LDB_ERR_ENTRY_ALREADY_EXISTS;
+	case MDB_NOTFOUND:
+	case ENOENT:
+		return LDB_ERR_NO_SUCH_OBJECT;
+	case EACCES:
+		return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+	default:
+		break;
+	}
+	return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+			 int ecode,
+			 const char *file,
+			 int line)
+{
+	int ldb_err = ldb_mdb_err_map(ecode);
+	char *reason = mdb_strerror(ecode);
+	ldb_asprintf_errstring(ldb,
+			       "(%d) - %s at %s:%d",
+			       ecode,
+			       reason,
+			       file,
+			       line);
+	return ldb_err;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+	if (ltx == NULL) {
+		return NULL;
+	}
+
+	return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+	if (lmdb->txlist) {
+		talloc_steal(lmdb->txlist, ltx);
+	}
+
+	DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+	DLIST_REMOVE(lmdb->txlist, ltx);
+	talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+	struct lmdb_trans *ltx;
+
+	ltx = lmdb->txlist;
+	return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+	MDB_txn *txn;
+	if (lmdb->read_txn != NULL) {
+		return lmdb->read_txn;
+	}
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		int ret;
+		ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+		if (ret != 0) {
+			lmdb->error = ret;
+			ldb_asprintf_errstring(lmdb->ldb,
+					       "%s failed: %s\n", __FUNCTION__,
+					       mdb_strerror(ret));
+		}
+		lmdb->read_txn = txn;
+	}
+	return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+		      struct ldb_val key,
+		      struct ldb_val data, int flags)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	int mdb_flags;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+	mdb_data.mv_size = data.length;
+	mdb_data.mv_data = data.data;
+
+	if (flags == TDB_INSERT) {
+		mdb_flags = MDB_NOOVERWRITE;
+	} else if ((flags == TDB_MODIFY)) {
+		/*
+		 * Modifying a record, ensure that it exists.
+		 * This mimics the TDB semantics
+		 */
+		MDB_val value;
+		lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+		if (lmdb->error != MDB_SUCCESS) {
+			if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+				mdb_txn_commit(lmdb->read_txn);
+				lmdb->read_txn = NULL;
+			}
+			return ldb_mdb_error(lmdb->ldb, lmdb->error);
+		}
+		mdb_flags = 0;
+	} else {
+		mdb_flags = 0;
+	}
+
+        lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+        lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+		            ldb_kv_traverse_fn fn,
+			    void *ctx)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+	MDB_cursor *cursor = NULL;
+	int ret;
+
+	txn = get_current_txn(lmdb);
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+	if (lmdb->error != MDB_SUCCESS) {
+		goto done;
+	}
+
+	while ((lmdb->error = mdb_cursor_get(
+			cursor, &mdb_key,
+			&mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+		struct ldb_val key = {
+			.length = mdb_key.mv_size,
+			.data = mdb_key.mv_data,
+		};
+		struct ldb_val data = {
+			.length = mdb_data.mv_size,
+			.data = mdb_data.mv_data,
+		};
+
+		ret = fn(ltdb, key, data, ctx);
+		if (ret != 0) {
+			goto done;
+		}
+	}
+	if (lmdb->error == MDB_NOTFOUND) {
+		lmdb->error = MDB_SUCCESS;
+	}
+done:
+	if (cursor != NULL) {
+		mdb_cursor_close(cursor);
+	}
+
+	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+	}
+
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+				  struct ldb_val key,
+				  struct ldb_val key2,
+				  struct ldb_val data,
+				  void *state)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	struct ldb_val copy;
+	int ret = LDB_SUCCESS;
+
+	/*
+	 * Need to take a copy of the data as the delete operation alters the
+	 * data, as it is in private lmdb memory.
+	 */
+	copy.length = data.length;
+	copy.data = talloc_memdup(ltdb, data.data, data.length);
+	if (copy.data == NULL) {
+		lmdb->error = MDB_PANIC;
+		return ldb_oom(lmdb->ldb);
+	}
+
+	lmdb->error = lmdb_delete(ltdb, key);
+	if (lmdb->error != MDB_SUCCESS) {
+		ldb_debug(
+			lmdb->ldb,
+			LDB_DEBUG_ERROR,
+			"Failed to delete %*.*s "
+			"for rekey as %*.*s: %s",
+			(int)key.length, (int)key.length,
+			(const char *)key.data,
+			(int)key2.length, (int)key2.length,
+			(const char *)key.data,
+			mdb_strerror(lmdb->error));
+		ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+		goto done;
+	}
+	lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+	if (lmdb->error != MDB_SUCCESS) {
+		ldb_debug(
+			lmdb->ldb,
+			LDB_DEBUG_ERROR,
+			"Failed to rekey %*.*s as %*.*s: %s",
+			(int)key.length, (int)key.length,
+			(const char *)key.data,
+			(int)key2.length, (int)key2.length,
+			(const char *)key.data,
+			mdb_strerror(lmdb->error));
+		ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+		goto done;
+	}
+
+done:
+	if (copy.data != NULL) {
+		TALLOC_FREE(copy.data);
+		copy.length = 0;
+	}
+
+	/*
+	 * Explicity invalidate the data, as the delete has done this
+	 */
+	data.length = 0;
+	data.data = NULL;
+
+	return ret;
+}
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+			     int (*parser)(struct ldb_val key, struct ldb_val data,
+					   void *private_data),
+			     void *ctx)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi;
+	struct ldb_val data;
+
+	txn = get_current_txn(lmdb);
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction active");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+        lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+	if (lmdb->error != MDB_SUCCESS) {
+		/* TODO closing a handle should not even be necessary */
+		mdb_dbi_close(lmdb->env, dbi);
+		if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+			mdb_txn_commit(lmdb->read_txn);
+			lmdb->read_txn = NULL;
+		}
+		if (lmdb->error == MDB_NOTFOUND) {
+			return LDB_ERR_NO_SUCH_OBJECT;
+		}
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	data.data = mdb_data.mv_data;
+	data.length = mdb_data.mv_size;
+
+	/* TODO closing a handle should not even be necessary */
+	mdb_dbi_close(lmdb->env, dbi);
+
+	/* We created a read transaction, commit it */
+	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+	}
+	return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+	void *data = ldb_module_get_private(module);
+	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	lmdb->error = MDB_SUCCESS;
+	if (ltdb->in_transaction == 0 &&
+	    ltdb->read_lock_count == 0) {
+		lmdb->error = mdb_txn_begin(lmdb->env,
+					    NULL,
+					    MDB_RDONLY,
+					    &lmdb->read_txn);
+	}
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	ltdb->read_lock_count++;
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+	void *data = ldb_module_get_private(module);
+	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+		struct lmdb_private *lmdb = ltdb->lmdb_private;
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+		ltdb->read_lock_count--;
+		return LDB_SUCCESS;
+	}
+	ltdb->read_lock_count--;
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	struct lmdb_trans *ltx;
+	struct lmdb_trans *ltx_head;
+	MDB_txn *tx_parent;
+
+	ltx = talloc_zero(lmdb, struct lmdb_trans);
+	if (ltx == NULL) {
+		return ldb_oom(lmdb->ldb);
+	}
+
+	ltx_head = lmdb_private_trans_head(lmdb);
+
+	tx_parent = lmdb_trans_get_tx(ltx_head);
+
+	lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, &ltx->tx);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	trans_push(lmdb, ltx);
+
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+	struct lmdb_trans *ltx;
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	ltx = lmdb_private_trans_head(lmdb);
+	if (ltx == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	mdb_txn_abort(ltx->tx);
+	trans_finished(lmdb, ltx);
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+	/* No need to prepare a commit */
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+	struct lmdb_trans *ltx;
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	ltx = lmdb_private_trans_head(lmdb);
+	if (ltx == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	lmdb->error = mdb_txn_commit(ltx->tx);
+	trans_finished(lmdb, ltx);
+
+	return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+	return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+	return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+	return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+	/*
+	 * lmdb does no provide a quick way to determine if the database
+	 * has changed.  This function always returns true.
+	 *
+	 * Note that tdb uses a sequence number that allows this function
+	 * to be implemented efficiently.
+	 */
+	return true;
+}
+
+
+static struct kv_db_ops lmdb_key_value_ops = {
+	.store             = lmdb_store,
+	.delete            = lmdb_delete,
+	.iterate           = lmdb_traverse_fn,
+	.update_in_iterate = lmdb_update_in_iterate,
+	.fetch_and_parse   = lmdb_parse_record,
+	.lock_read         = lmdb_lock_read,
+	.unlock_read       = lmdb_unlock_read,
+	.begin_write       = lmdb_transaction_start,
+	.prepare_write     = lmdb_transaction_prepare_commit,
+	.finish_write      = lmdb_transaction_commit,
+	.abort_write       = lmdb_transaction_cancel,
+	.error             = lmdb_error,
+	.errorstr          = lmdb_errorstr,
+	.name              = lmdb_name,
+	.has_changed       = lmdb_changed,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+	const char *path;
+
+	/* parse the url */
+	if (strchr(url, ':')) {
+		if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+			return NULL;
+		}
+		path = url + MDB_URL_PREFIX_SIZE;
+	} else {
+		path = url;
+	}
+
+	return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+	struct lmdb_trans *ltx = NULL;
+
+	/*
+	 * Close the read transaction if it's open
+	 */
+	if (lmdb->read_txn != NULL) {
+		mdb_txn_abort(lmdb->read_txn);
+	}
+
+	if (lmdb->env == NULL) {
+		return 0;
+	}
+
+	/*
+	 * Abort any currently active transactions
+	 */
+	ltx = lmdb_private_trans_head(lmdb);
+	while (ltx != NULL) {
+		mdb_txn_abort(ltx->tx);
+		trans_finished(lmdb, ltx);
+		ltx = lmdb_private_trans_head(lmdb);
+	}
+
+	mdb_env_close(lmdb->env);
+	lmdb->env = NULL;
+
+	return 0;
+}
+
+static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
+					    struct ldb_context *ldb,
+					    const char *path)
+{
+	struct lmdb_private *lmdb;
+	int ret;
+
+	lmdb = talloc_zero(ldb, struct lmdb_private);
+	if (lmdb == NULL) {
+		return NULL;
+	}
+	lmdb->ldb = ldb;
+
+	ret = mdb_env_create(&lmdb->env);
+	if (ret != 0) {
+		ldb_asprintf_errstring(ldb,
+				"Could not create MDB environment %s: %s\n",
+				path, mdb_strerror(ret));
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	/* Close when lmdb is released */
+	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+	if (ret != 0) {
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	mdb_env_set_maxreaders(lmdb->env, 100000);
+	/* MDB_NOSUBDIR implies there is a separate file called path and a
+	 * separate lockfile called path-lock
+	 */
+	ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+	if (ret != 0) {
+		ldb_asprintf_errstring(ldb,
+				"Could not open DB %s: %s\n",
+				path, mdb_strerror(ret));
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	return lmdb;
+
+}
+
+static int lmdb_connect(struct ldb_context *ldb, const char *url,
+			unsigned int flags, const char *options[],
+			struct ldb_module **_module)
+{
+	const char *path = NULL;
+	struct lmdb_private *lmdb = NULL;
+	struct ltdb_private *ltdb = NULL;
+
+        /*
+         * We hold locks, so we must use a private event context
+         * on each returned handle
+         */
+        ldb_set_require_private_event_context(ldb);
+
+	path = lmdb_get_path(url);
+	if (path == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+        ltdb = talloc_zero(ldb, struct ltdb_private);
+        if (!ltdb) {
+                ldb_oom(ldb);
+                return LDB_ERR_OPERATIONS_ERROR;
+        }
+	ltdb->kv_ops = &lmdb_key_value_ops;
+
+	lmdb = lmdb_pvt_create(ldb, ldb, path);
+	if (lmdb == NULL) {
+		ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ltdb->lmdb_private = lmdb;
+	if (flags & LDB_FLG_RDONLY) {
+		ltdb->read_only = true;
+	}
+        return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
+
+
+
+int ldb_mdb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 00000000000..18d28e822d7
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,57 @@
+/*
+   ldb database library using mdb back end - transaction operations
+
+   Copyright (C) Jakub Hrozek 2015
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include <lmdb.h>
+
+#include "ldb_private.h"
+
+
+struct lmdb_private {
+	struct ldb_context *ldb;
+	MDB_env *env;
+
+	struct lmdb_trans *txlist;
+
+	struct ldb_mdb_metadata {
+		struct ldb_message *attributes;
+		unsigned seqnum;
+	} *meta;
+	int error;
+	MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+	struct lmdb_trans *next;
+	struct lmdb_trans *prev;
+
+	MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f14666ba88a..7d92804a8e9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
 struct ltdb_private {
 	const struct kv_db_ops *kv_ops;
 	TDB_CONTEXT *tdb;
+	struct lmdb_private *lmdb_private;
 	unsigned int connect_flags;
 	
 	unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59eca4..2da2ca8ec70 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
 #include <ldb.h>
 #include <ldb_private.h>
 
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
 static struct ldb_context *ldb;
 bool show_index = false;
 bool validate_contents = false;
@@ -166,6 +171,116 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
 	return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
 }
 
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+	int ret;
+	struct MDB_env *env = NULL;
+	struct MDB_txn *txn = NULL;
+	MDB_dbi dbi;
+	struct MDB_cursor *cursor = NULL;
+	struct MDB_val key;
+	struct MDB_val data;
+
+	ret = mdb_env_create(&env);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not create MDB environment: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_env_open(env,
+			   fname,
+			   MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY,
+			   0600);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open environment for %s: (%d)  %s\n",
+			fname,
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not start transaction: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open database: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_txn;
+	}
+
+	ret = mdb_cursor_open(txn, dbi, &cursor);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open cursor: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_txn;
+	}
+
+	ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+	if (ret != 0 && ret != MDB_NOTFOUND) {
+		fprintf(stderr,
+			"Could not find first record: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_cursor;
+	}
+	while (ret != MDB_NOTFOUND) {
+		struct TDB_DATA tkey = {
+			.dptr = key.mv_data,
+			.dsize = key.mv_size
+		};
+		struct TDB_DATA tdata = {
+			.dptr = data.mv_data,
+			.dsize = data.mv_size
+		};
+		traverse_fn(NULL, tkey, tdata, dn);
+		ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+		if (ret != 0 && ret != MDB_NOTFOUND) {
+			fprintf(stderr,
+				"Could not read next record: (%d)  %s\n",
+				ret,
+				mdb_strerror(ret));
+			goto close_cursor;
+		}
+	}
+	ret = 0;
+
+close_cursor:
+	mdb_cursor_close(cursor);
+close_txn:
+	mdb_txn_commit(txn);
+close_env:
+	mdb_env_close(env);
+
+	if (ret != 0) {
+		return 1;
+	}
+	return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+	/* not built with lmdb support */
+	return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
 static void usage( void)
 {
 	printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +344,14 @@ static void usage( void)
 
 	fname = argv[optind];
 
-	return dump_tdb(fname, dn, emergency);
+	rc = dump_lmdb(fname, dn, emergency);
+	if (rc != 0) {
+		rc = dump_tdb(fname, dn, emergency);
+		if (rc != 0) {
+			fprintf(stderr, "Failed to open %s\n", fname);
+			return 1;
+		}
+	}
+	return 0;
+
 }
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 81fcb55e79b..fc216b0831f 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
     srcdir = srcdir + '/..'
 sys.path.insert(0, srcdir + '/buildtools/wafsamba')
 
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
 
 samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
                         lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -92,6 +92,16 @@ def configure(conf):
                                          implied_deps='replace talloc tdb tevent'):
                 conf.define('USING_SYSTEM_LDB', 1)
 
+    if conf.env.standalone_ldb:
+        # Require lmdb support for standalone mode.
+        conf.env.REQUIRE_LMDB = True
+    elif not Options.options.without_ad_dc:
+        # Require lmdb support for addc mode
+        conf.env.REQUIRE_LMDB = True
+    else:
+        conf.env.REQUIRE_LMDB = False
+
+
     if conf.CONFIG_SET('USING_SYSTEM_LDB'):
         v = VERSION.split('.')
         conf.DEFINE('EXPECTED_SYSTEM_LDB_VERSION_MAJOR', int(v[0]))
@@ -110,6 +120,36 @@ def configure(conf):
         if not sys.platform.startswith("openbsd"):
             conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
 
+    # if lmdb support is enabled then we require lmdb
+    # is present, build the mdb back end and enable lmdb support in
+    # the tools.
+    if conf.env.REQUIRE_LMDB:
+        if not conf.CHECK_CFG(package='lmdb',
+                              args='"lmdb >= 0.9.16" --cflags --libs',
+                              msg='Checking for lmdb >= 0.9.16',
+                              mandatory=False):
+            if not conf.CHECK_CODE('''
+                    #if MDB_VERSION_MAJOR == 0 \
+                      && MDB_VERSION_MINOR <= 9 \
+                      && MDB_VERSION_PATCH < 16
+                    #error LMDB too old
+                    #endif
+                    ''',
+                    'HAVE_GOOD_LMDB_VERSION',
+                    headers='lmdb.h',
+                    msg='Checking for lmdb >= 0.9.16 via header check'):
+
+                if conf.env.standalone_ldb:
+                    raise Utils.WafError('ldb requires '
+                                         'lmdb 0.9.16 or later')
+                elif not Options.options.without_ad_dc:
+                    raise Utils.WafError('Samba AD DC requires '
+                                         'lmdb 0.9.16 or later')
+
+        if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+            conf.DEFINE('HAVE_LMDB', '1')
+            conf.env.ENABLE_MDB_BACKEND = True
+
     conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
 
     conf.SAMBA_CONFIG_H()
@@ -321,6 +361,20 @@ def build(bld):
                           private_library=True,
                           deps='tdb ldb')
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            bld.SAMBA_MODULE('ldb_mdb',
+                             bld.SUBDIR('ldb_mdb',
+                                         '''ldb_mdb.c '''),
+                             init_function='ldb_mdb_init',
+                             module_init_name='ldb_init_module',
+                             internal_module=False,
+                             deps='ldb ldb_key_value lmdb',
+                             subsystem='ldb')
+            lmdb_deps = ' ldb_mdb_int'
+        else:
+            lmdb_deps = ''
+
+
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
@@ -338,8 +392,14 @@ def build(bld):
         bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
                          install=False)
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            lmdb_deps = ' lmdb'
+        else:
+            lmdb_deps = ''
         # ldbdump doesn't get installed
-        bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+        bld.SAMBA_BINARY('ldbdump',
+                         'tools/ldbdump.c',
+                         deps='ldb-cmdline ldb' + lmdb_deps,
                          install=False)
 
         bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -365,6 +425,18 @@ def build(bld):
                          deps='cmocka ldb',
                          install=False)
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+                             source='tests/ldb_mod_op_test.c',
+                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
+                             deps='cmocka ldb',
+                             install=False)
+            bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
+                             source='tests/ldb_kv_ops_test.c',
+                             cflags='-DTEST_BE=\"mdb\"',
+                             deps='cmocka ldb',
+                             install=False)
+
         bld.SAMBA_BINARY('ldb_msg_test',
                          source='tests/ldb_msg.c',
                          deps='cmocka ldb',
@@ -397,10 +469,15 @@ def test(ctx):
     print("Python testsuite returned %d" % pyret)
 
     cmocka_ret = 0
-    for test_exe in ['ldb_tdb_mod_op_test',
-                     'ldb_tdb_guid_mod_op_test',
-                     'ldb_msg_test',
-                     'ldb_tdb_kv_ops_test']:
+    test_exes = ['ldb_tdb_mod_op_test',
+                 'ldb_tdb_guid_mod_op_test',
+                 'ldb_msg_test',
+                 'ldb_tdb_kv_ops_test']
+
+    if env.ENABLE_MDB_BACKEND:
+        test_exes.append('ldb_mdb_mod_op_test')
+        test_exes.append('ldb_mdb_kv_ops_test')
+    for test_exe in test_exes:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
             cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
 
-- 
2.11.0


From bb767ff21c87b442048cc9c02e4b9f23aca1b7db Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 31/54] ldb: Handle multiple ldb url schemes

Here we use ldb_relative_path more often and allow for the stripping of
ldb:// and mdb://.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb-samba/ldb_wrap.c                            |  6 ++++++
 source4/dsdb/samdb/ldb_modules/partition_metadata.c | 20 +++++++++-----------
 source4/dsdb/samdb/ldb_modules/schema_load.c        | 19 +++++--------------
 3 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 143e128d5d7..4dc5ca6da2a 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -356,6 +356,12 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 	if (strncmp("tdb://", base_url, 6) == 0) {
 		base_url = base_url+6;
 	}
+	else if (strncmp("mdb://", base_url, 6) == 0) {
+		base_url = base_url+6;
+	}
+	else if (strncmp("ldb://", base_url, 6) == 0) {
+		base_url = base_url+6;
+	}
 	path = talloc_strdup(mem_ctx, base_url);
 	if (path == NULL) {
 		return NULL;
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index e3ad0d8c6c2..510228e319b 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -18,6 +18,7 @@
 */
 
 #include "dsdb/samdb/ldb_modules/partition.h"
+#include "lib/ldb-samba/ldb_wrap.h"
 #include "system/filesys.h"
 
 #define LDB_METADATA_SEQ_NUM	"SEQ_NUM"
@@ -185,7 +186,6 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 	TALLOC_CTX *tmp_ctx;
 	struct partition_private_data *data;
 	struct loadparm_context *lp_ctx;
-	const char *sam_name;
 	char *filename, *dirname;
 	int open_flags;
 	struct stat statbuf;
@@ -202,15 +202,11 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 		return ldb_module_oom(module);
 	}
 
-	sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
-	if (!sam_name) {
-		talloc_free(tmp_ctx);
-		return ldb_operr(ldb);
-	}
-	if (strncmp("tdb://", sam_name, 6) == 0) {
-		sam_name += 6;
-	}
-	filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
+	/* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+	filename = ldb_relative_path(ldb,
+				     tmp_ctx,
+				     "sam.ldb.d/metadata.tdb");
+
 	if (!filename) {
 		talloc_free(tmp_ctx);
 		return ldb_oom(ldb);
@@ -222,7 +218,9 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 
 		/* While provisioning, sam.ldb.d directory may not exist,
 		 * so create it. Ignore errors, if it already exists. */
-		dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name);
+		dirname = ldb_relative_path(ldb,
+					    tmp_ctx,
+					    "sam.ldb.d");
 		if (!dirname) {
 			talloc_free(tmp_ctx);
 			return ldb_oom(ldb);
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 2099fac1159..e21a5a5a7e2 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -32,6 +32,7 @@
 #include <tdb.h>
 #include "lib/tdb_wrap/tdb_wrap.h"
 #include "dsdb/samdb/ldb_modules/util.h"
+#include "lib/ldb-samba/ldb_wrap.h"
 
 #include "system/filesys.h"
 struct schema_load_private_data {
@@ -58,7 +59,6 @@ static int schema_metadata_open(struct ldb_module *module)
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	TALLOC_CTX *tmp_ctx;
 	struct loadparm_context *lp_ctx;
-	const char *sam_name;
 	char *filename;
 	int open_flags;
 	struct stat statbuf;
@@ -74,19 +74,10 @@ static int schema_metadata_open(struct ldb_module *module)
 		return ldb_module_oom(module);
 	}
 
-	sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
-	if (!sam_name) {
-		talloc_free(tmp_ctx);
-		return ldb_operr(ldb);
-	}
-	if (strncmp("tdb://", sam_name, 6) == 0) {
-		sam_name += 6;
-	}
-	filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
-	if (!filename) {
-		talloc_free(tmp_ctx);
-		return ldb_oom(ldb);
-	}
+	/* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+	filename = ldb_relative_path(ldb,
+				     tmp_ctx,
+				     "sam.ldb.d/metadata.tdb");
 
 	open_flags = O_RDWR;
 	if (stat(filename, &statbuf) != 0) {
-- 
2.11.0


From 9916a587dfeff3535116c39fed2bdf925e4bb988 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 32/54] ldb: Introduce new ldb:// prefix to allow failover

If you try to open with tdb first you find that you can clobber a
database due to the default tdb behaviour.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_ldb/ldb_ldb.c      | 78 +++++++++++++++++++++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.c      | 79 +++++++++++++++++++++++++-----------------
 lib/ldb/ldb_mdb/ldb_mdb.h      |  3 ++
 lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 +++++++++++++++++
 lib/ldb/wscript                | 19 ++++++++--
 5 files changed, 176 insertions(+), 34 deletions(-)
 create mode 100644 lib/ldb/ldb_ldb/ldb_ldb.c
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c

diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
new file mode 100644
index 00000000000..a63967ff8d6
--- /dev/null
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -0,0 +1,78 @@
+/*
+ * ldb connection and module initialisation
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#include "ldb_private.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+
+/*
+  connect to the database
+*/
+static int lldb_connect(struct ldb_context *ldb,
+			const char *url,
+			unsigned int flags,
+			const char *options[],
+			struct ldb_module **module)
+{
+	const char *path;
+	int ret;
+
+	/*
+	 * Check and remove the url prefix
+	 */
+	if (strchr(url, ':')) {
+		if (strncmp(url, "ldb://", 6) != 0) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR,
+				  "Invalid ldb URL '%s'", url);
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		path = url+6;
+	} else {
+		path = url;
+	}
+
+	/*
+	 * Don't create the database if it's not there
+	 */
+	flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+	/*
+	 * Try opening the database as an lmdb
+	 */
+	ret = lmdb_connect(ldb, path, flags, options, module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (ret == LDB_ERR_UNAVAILABLE) {
+		/*
+		 * Not mdb so try as tdb
+		 */
+		ret = ltdb_connect(ldb, path, flags, options, module);
+	}
+#else
+	ret = ltdb_connect(ldb, path, flags, options, module);
+#endif
+	return ret;
+}
+
+int ldb_ldb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("ldb", lldb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 8bc82715390..314c78ff626 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -614,26 +614,30 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 	return 0;
 }
 
-static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
-					    struct ldb_context *ldb,
-					    const char *path)
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+				  struct ldb_context *ldb,
+				  const char *path,
+				  unsigned int flags,
+				  struct lmdb_private *lmdb)
 {
-	struct lmdb_private *lmdb;
 	int ret;
+	unsigned int mdb_flags;
 
-	lmdb = talloc_zero(ldb, struct lmdb_private);
-	if (lmdb == NULL) {
-		return NULL;
+	if (flags & LDB_FLG_DONT_CREATE_DB) {
+		struct stat st;
+		if (stat(path, &st) != 0) {
+			return LDB_ERR_UNAVAILABLE;
+		}
 	}
-	lmdb->ldb = ldb;
 
 	ret = mdb_env_create(&lmdb->env);
 	if (ret != 0) {
-		ldb_asprintf_errstring(ldb,
-				"Could not create MDB environment %s: %s\n",
-				path, mdb_strerror(ret));
-		talloc_free(lmdb);
-		return NULL;
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not create MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
 	/* Close when lmdb is released */
@@ -641,34 +645,45 @@ static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
 
 	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
 	if (ret != 0) {
-		talloc_free(lmdb);
-		return NULL;
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not open MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return ldb_mdb_err_map(ret);
 	}
 
 	mdb_env_set_maxreaders(lmdb->env, 100000);
 	/* MDB_NOSUBDIR implies there is a separate file called path and a
 	 * separate lockfile called path-lock
 	 */
-	ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+	mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+	if (flags & LDB_FLG_RDONLY) {
+		mdb_flags |= MDB_RDONLY;
+	}
+	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
 				"Could not open DB %s: %s\n",
 				path, mdb_strerror(ret));
 		talloc_free(lmdb);
-		return NULL;
+		return ldb_mdb_err_map(ret);
 	}
 
-	return lmdb;
+	return LDB_SUCCESS;
 
 }
 
-static int lmdb_connect(struct ldb_context *ldb, const char *url,
-			unsigned int flags, const char *options[],
-			struct ldb_module **_module)
+int lmdb_connect(struct ldb_context *ldb,
+		 const char *url,
+		 unsigned int flags,
+		 const char *options[],
+		 struct ldb_module **_module)
 {
 	const char *path = NULL;
 	struct lmdb_private *lmdb = NULL;
 	struct ltdb_private *ltdb = NULL;
+	int ret;
 
         /*
          * We hold locks, so we must use a private event context
@@ -687,12 +702,19 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
                 ldb_oom(ldb);
                 return LDB_ERR_OPERATIONS_ERROR;
         }
-	ltdb->kv_ops = &lmdb_key_value_ops;
 
-	lmdb = lmdb_pvt_create(ldb, ldb, path);
+	lmdb = talloc_zero(ldb, struct lmdb_private);
 	if (lmdb == NULL) {
-		ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
-		return LDB_ERR_OPERATIONS_ERROR;
+		TALLOC_FREE(ltdb);
+                ldb_oom(ldb);
+                return LDB_ERR_OPERATIONS_ERROR;
+	}
+	lmdb->ldb = ldb;
+	ltdb->kv_ops = &lmdb_key_value_ops;
+
+	ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+	if (ret != LDB_SUCCESS) {
+		return ret;
 	}
 
 	ltdb->lmdb_private = lmdb;
@@ -702,10 +724,3 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
         return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
 }
 
-
-
-int ldb_mdb_init(const char *version)
-{
-	LDB_MODULE_CHECK_VERSION(version);
-	return ldb_register_backend("mdb", lmdb_connect, false);
-}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 18d28e822d7..e62e353b54e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -53,5 +53,8 @@ struct lmdb_trans {
 };
 
 int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+		 unsigned int flags, const char *options[],
+		 struct ldb_module **_module);
 
 #endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 00000000000..339c3f22b2a
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+   ldb database library using mdb back end
+
+   Copyright (C) Jakub Hrozek 2014
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index fc216b0831f..ec5d8eff97f 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -364,17 +364,32 @@ def build(bld):
         if bld.CONFIG_SET('HAVE_LMDB'):
             bld.SAMBA_MODULE('ldb_mdb',
                              bld.SUBDIR('ldb_mdb',
-                                         '''ldb_mdb.c '''),
+                                        '''ldb_mdb_init.c'''),
                              init_function='ldb_mdb_init',
                              module_init_name='ldb_init_module',
                              internal_module=False,
-                             deps='ldb ldb_key_value lmdb',
+                             deps='ldb ldb_key_value ldb_mdb_int',
                              subsystem='ldb')
+
+            bld.SAMBA_LIBRARY('ldb_mdb_int',
+                              bld.SUBDIR('ldb_mdb',
+                                         '''ldb_mdb.c '''),
+                              private_library=True,
+                              deps='ldb lmdb ldb_key_value')
             lmdb_deps = ' ldb_mdb_int'
         else:
             lmdb_deps = ''
 
 
+        bld.SAMBA_MODULE('ldb_ldb',
+                         bld.SUBDIR('ldb_ldb',
+                                    '''ldb_ldb.c'''),
+                         init_function='ldb_ldb_init',
+                         module_init_name='ldb_init_module',
+                         internal_module=False,
+                         deps='ldb ldb_key_value' + lmdb_deps,
+                         subsystem='ldb')
+
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
-- 
2.11.0


From 18937d9c6460a55a4064682134175d001ba71d5c Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 5 Jan 2018 09:28:54 +1300
Subject: [PATCH 33/54] tests: Replace some references to tdb with ldb://

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/encrypted_secrets.py      |  2 +-
 source4/torture/drs/python/samba_tool_drs.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/python/samba/tests/encrypted_secrets.py b/python/samba/tests/encrypted_secrets.py
index 3b6934f0ccb..114f3b91108 100644
--- a/python/samba/tests/encrypted_secrets.py
+++ b/python/samba/tests/encrypted_secrets.py
@@ -54,7 +54,7 @@ class EncryptedSecretsTests(TestCase):
         backend_subpath = os.path.join("sam.ldb.d",
                                        backend_filename)
         backend_path = self.lp.private_path(backend_subpath)
-        backenddb = ldb.Ldb(backend_path)
+        backenddb = ldb.Ldb("ldb://" + backend_path, flags=ldb.FLG_DONT_CREATE_DB)
 
         dn = "CN=Administrator,CN=Users,%s" % basedn
 
diff --git a/source4/torture/drs/python/samba_tool_drs.py b/source4/torture/drs/python/samba_tool_drs.py
index fcc681dc9e4..502c8096603 100644
--- a/source4/torture/drs/python/samba_tool_drs.py
+++ b/source4/torture/drs/python/samba_tool_drs.py
@@ -282,7 +282,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
                                    self.dc1,
                                    self.cmdline_creds,
                                    self.tempdir))
-        ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+        ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
         nc_name = ldb_rootdse["defaultNamingContext"]
         ds_name = ldb_rootdse["dsServiceName"]
         ldap_service_name = str(server_rootdse["ldapServiceName"][0])
@@ -291,7 +291,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
         self.assertEqual(ds_name, server_ds_name)
         self.assertEqual(ldap_service_name, server_ldap_service_name)
 
-        samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+        samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
                                           ldap_only=False, lp=self.get_loadparm())
         def get_krbtgt_pw():
             krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
@@ -346,13 +346,13 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
                                    self.dc1,
                                    self.cmdline_creds,
                                    self.tempdir))
-        ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+        ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
         nc_name = ldb_rootdse["defaultNamingContext"]
         config_nc_name = ldb_rootdse["configurationNamingContext"]
         ds_name = ldb_rootdse["dsServiceName"]
         ldap_service_name = str(server_rootdse["ldapServiceName"][0])
 
-        samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+        samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
                                           ldap_only=False, lp=self.get_loadparm())
         krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
         self.assertIsNotNone(krbtgt_pw)
@@ -381,7 +381,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
         self.assertRaises(samba.tests.BlackboxProcessError, demote_self)
 
         # While we have this cloned, try demoting the other server on the clone
-        out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H %s/private/sam.ldb"
+        out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H ldb://%s/private/sam.ldb"
                                 % (self.dc2,
                                    self.tempdir))
 
-- 
2.11.0


From 6ca5da26bdb4a2e061f78aa0bce8f554e9495e3f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 12:40:01 +1300
Subject: [PATCH 34/54] tests/dlz_bind9: support for multiple db types

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/torture/dns/dlz_bind9.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source4/torture/dns/dlz_bind9.c b/source4/torture/dns/dlz_bind9.c
index 2234e7a23f6..42b104e070c 100644
--- a/source4/torture/dns/dlz_bind9.c
+++ b/source4/torture/dns/dlz_bind9.c
@@ -58,7 +58,7 @@ static char *test_dlz_bind9_binddns_dir(struct torture_context *tctx,
 					const char *file)
 {
 	return talloc_asprintf(tctx,
-			       "%s/%s",
+			       "ldb://%s/%s",
 			       lpcfg_binddns_dir(tctx->lp_ctx),
 			       file);
 }
-- 
2.11.0


From b3f047d496df248889e4326f6d45f0af85a34b21 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 35/54] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb

This is used in selftest with 'ldb:nosync = true'.

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 314c78ff626..a34cf35a7d5 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -661,6 +661,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 	if (flags & LDB_FLG_RDONLY) {
 		mdb_flags |= MDB_RDONLY;
 	}
+	if (flags & LDB_FLG_NOSYNC) {
+		mdb_flags |= MDB_NOSYNC;
+	}
 	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
-- 
2.11.0


From 64bdc3e018f70aa3380fecfc7816b57cd4984d8d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 36/54] tests: Add tests to check for max key length and DB
 size

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_lmdb_size_test.c | 213 ++++++++++++++++++++++++++++++++
 lib/ldb/tests/ldb_lmdb_test.c      | 243 +++++++++++++++++++++++++++++++++++++
 lib/ldb/wscript                    |  12 ++
 3 files changed, 468 insertions(+)
 create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
 create mode 100644 lib/ldb/tests/ldb_lmdb_test.c

diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 00000000000..506c586289e
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,213 @@
+/*
+ * lmdb backend specific tests for ldb
+ * Tests for truncated index keys
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE  "mdb"
+
+struct ldbtest_ctx {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+
+	const char *dbfile;
+	const char *lockfile;   /* lockfile is separate */
+
+	const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+	int ret;
+
+	errno = 0;
+	ret = unlink(test_ctx->lockfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+
+	errno = 0;
+	ret = unlink(test_ctx->dbfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+
+	test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+	assert_non_null(test_ctx);
+
+	test_ctx->ev = tevent_context_init(test_ctx);
+	assert_non_null(test_ctx->ev);
+
+	test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+	assert_non_null(test_ctx->ldb);
+
+	test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+	assert_non_null(test_ctx->dbfile);
+
+	test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+					     test_ctx->dbfile);
+	assert_non_null(test_ctx->lockfile);
+
+	test_ctx->dbpath = talloc_asprintf(test_ctx,
+			TEST_BE"://%s", test_ctx->dbfile);
+	assert_non_null(test_ctx->dbpath);
+
+	unlink_old_db(test_ctx);
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+
+	unlink_old_db(test_ctx);
+	talloc_free(test_ctx);
+	return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	ldbtest_noconn_teardown((void **) &test_ctx);
+	return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+	int ret, x;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	const int MB = 1024 * 1024;
+	char *blob = NULL;
+
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+
+	blob = talloc_zero_size(tmp_ctx, (MB + 1));
+	assert_non_null(blob);
+	memset(blob, 'x', MB);
+
+
+	for (x = 0; x < 6144; x++) {
+		msg = ldb_msg_new(tmp_ctx);
+		assert_non_null(msg);
+
+		msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+		assert_non_null(msg->dn);
+
+		ldb_transaction_start(test_ctx->ldb);
+		ret = ldb_msg_add_string(msg, "blob", blob);
+		assert_int_equal(ret, 0);
+
+		ret = ldb_add(test_ctx->ldb, msg);
+		assert_int_equal(ret, 0);
+		ldb_transaction_commit(test_ctx->ldb);
+
+		TALLOC_FREE(msg);
+	}
+	talloc_free(tmp_ctx);
+	{
+		struct stat s;
+		ret = stat(test_ctx->dbfile, &s);
+		assert_int_equal(ret, 0);
+		assert_true(s.st_size > (6144LL * MB));
+	}
+}
+
+int main(int argc, const char **argv)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(
+			test_db_size_gt_4GB,
+			ldbtest_setup,
+			ldbtest_teardown),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 00000000000..2902eaf0533
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,243 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied  from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE  "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+
+	const char *dbfile;
+	const char *lockfile;   /* lockfile is separate */
+
+	const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+	int ret;
+
+	errno = 0;
+	ret = unlink(test_ctx->lockfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+
+	errno = 0;
+	ret = unlink(test_ctx->dbfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+
+	test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+	assert_non_null(test_ctx);
+
+	test_ctx->ev = tevent_context_init(test_ctx);
+	assert_non_null(test_ctx->ev);
+
+	test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+	assert_non_null(test_ctx->ldb);
+
+	test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+	assert_non_null(test_ctx->dbfile);
+
+	test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+					     test_ctx->dbfile);
+	assert_non_null(test_ctx->lockfile);
+
+	test_ctx->dbpath = talloc_asprintf(test_ctx,
+			TEST_BE"://%s", test_ctx->dbfile);
+	assert_non_null(test_ctx->dbpath);
+
+	unlink_old_db(test_ctx);
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+
+	unlink_old_db(test_ctx);
+	talloc_free(test_ctx);
+	return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	ldbtest_noconn_teardown((void **) &test_ctx);
+	return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_eq_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_gt_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index ec5d8eff97f..d9f47c798bf 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -446,6 +446,17 @@ def build(bld):
                              cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
                              deps='cmocka ldb',
                              install=False)
+            
+            bld.SAMBA_BINARY('ldb_lmdb_test',
+                             source='tests/ldb_lmdb_test.c',
+                             deps='cmocka ldb',
+                             install=False)
+
+            bld.SAMBA_BINARY('ldb_lmdb_size_test',
+                             source='tests/ldb_lmdb_size_test.c',
+                             deps='cmocka ldb',
+                             install=False)
+
             bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
                              source='tests/ldb_kv_ops_test.c',
                              cflags='-DTEST_BE=\"mdb\"',
@@ -491,6 +502,7 @@ def test(ctx):
 
     if env.ENABLE_MDB_BACKEND:
         test_exes.append('ldb_mdb_mod_op_test')
+        test_exes.append('ldb_lmdb_test')
         test_exes.append('ldb_mdb_kv_ops_test')
     for test_exe in test_exes:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
-- 
2.11.0


From e7f9f0cfd87d5d6dc2dc74419ebb8a27969dd70e Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 37/54] ldb_mdb: Store pid to change destructor on fork

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.h |  2 ++
 2 files changed, 24 insertions(+)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index a34cf35a7d5..711700c2697 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -587,6 +587,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 {
 	struct lmdb_trans *ltx = NULL;
 
+	/* Check if this is a forked child */
+	if (getpid() != lmdb->pid) {
+		int fd = 0;
+		/*
+		 * We cannot call mdb_env_close or commit any transactions,
+		 * otherwise they might appear finished in the parent.
+		 *
+		 */
+
+		if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+			close(fd);
+		}
+
+		/* Remove the pointer, so that no access should occur */
+		lmdb->env = NULL;
+
+		return 0;
+	}
+
 	/*
 	 * Close the read transaction if it's open
 	 */
@@ -673,6 +692,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 		return ldb_mdb_err_map(ret);
 	}
 
+	/* Store the original pid during the LMDB open */
+	lmdb->pid = getpid();
+
 	return LDB_SUCCESS;
 
 }
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index e62e353b54e..7e0729cc7af 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -43,6 +43,8 @@ struct lmdb_private {
 	int error;
 	MDB_txn *read_txn;
 
+	pid_t pid;
+
 };
 
 struct lmdb_trans {
-- 
2.11.0


From d531ce6f9c7c38cbca2803dffc741f672df0d86f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 38/54] lib ldb tests: Run api and index test on tdb and lmdb

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py   | 87 ++++++++++++++++++++++++++++++++++++++++++-
 lib/ldb/tests/python/index.py | 11 ++++++
 2 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 12409a8c991..9a2997f5b48 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -633,6 +633,16 @@ class SimpleLdb(LdbBaseTest):
         l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
 
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(SimpleLdbLmdb, self).setUp()
+
+    def tearDown(self):
+        super(SimpleLdbLmdb, self).tearDown()
+
 class SearchTests(LdbBaseTest):
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -1057,6 +1067,17 @@ class SearchTests(LdbBaseTest):
         self.assertEqual(len(res11), 1)
 
 
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(SearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(SearchTestsLmdb, self).tearDown()
+
+
 class IndexedSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1152,6 +1173,35 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
         self.IDXGUID = True
         self.IDXONE = True
 
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
 
 class AddModifyTests(LdbBaseTest):
     def tearDown(self):
@@ -1296,6 +1346,15 @@ class AddModifyTests(LdbBaseTest):
                     "objectUUID": b"0123456789abcde3"})
 
 
+class AddModifyTestsLmdb(AddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(AddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(AddModifyTestsLmdb, self).tearDown()
+
 class IndexedAddModifyTests(AddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1407,6 +1466,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
         self.l.transaction_commit()
         super(TransIndexedAddModifyTests, self).tearDown()
 
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
 
 class BadIndexTests(LdbBaseTest):
     def setUp(self):
@@ -1567,7 +1643,6 @@ class GUIDBadIndexTests(BadIndexTests):
 
         super(GUIDBadIndexTests, self).setUp()
 
-
 class DnTests(TestCase):
 
     def setUp(self):
@@ -2429,6 +2504,16 @@ class LdbResultTests(LdbBaseTest):
         self.assertEqual(got_pid, pid)
 
 
+class LdbResultTestsLmdb(LdbResultTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(LdbResultTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(LdbResultTestsLmdb, self).tearDown()
+
+
 class BadTypeTests(TestCase):
     def test_control(self):
         l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 9b9e4f3469f..3379fb9374f 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -1280,6 +1280,17 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
             code = e.args[0]
             self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
 
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
 if __name__ == '__main__':
     import unittest
     unittest.TestProgram()
-- 
2.11.0


From 285814ffe42c20d0ccea92ba93af79e3f2293b50 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:10 +1300
Subject: [PATCH 39/54] ldb: Allow tests to operate on ldb_mdb after new
 restrictions

The tests were working fine on ldb_mdb but only in the mode without
the GUID index.  This mode is not OK for production so has been banned
but this means we need to rework the tests to set an objectGUID on
each record.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++---------------
 1 file changed, 121 insertions(+), 64 deletions(-)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 9a2997f5b48..bb2c918e9f6 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -77,6 +77,10 @@ class SimpleLdb(LdbBaseTest):
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
         self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+        try:
+            self.ldb.add(self.index)
+        except AttributeError:
+            pass
 
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -165,6 +169,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo1")
         m["b"] = [b"a"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         self.assertRaises(ldb.LdbError, lambda: l.delete(m.dn, ["search_options:1:2"]))
         l.delete(m.dn)
@@ -177,6 +182,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo3")
         m["b"] = ["a"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             self.assertTrue(ldb.Dn(l, "dc=foo3") in l)
@@ -205,6 +211,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -245,6 +252,7 @@ class SimpleLdb(LdbBaseTest):
         m1 = ldb.Message()
         m1.dn = ldb.Dn(l, "dc=foo4")
         m1["bla"] = b"bla"
+        m1["objectUUID"] = b"0123456789abcdef"
         l.add(m1)
         try:
             s = l.search_iterator()
@@ -261,6 +269,7 @@ class SimpleLdb(LdbBaseTest):
             m2 = ldb.Message()
             m2.dn = ldb.Dn(l, "dc=foo5")
             m2["bla"] = b"bla"
+            m2["objectUUID"] = b"0123456789abcdee"
             l.add(m2)
 
             s = l.search_iterator()
@@ -317,6 +326,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = "bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -335,7 +345,8 @@ class SimpleLdb(LdbBaseTest):
     def test_add_dict(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
-             "bla": b"bla"}
+             "bla": b"bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -346,7 +357,8 @@ class SimpleLdb(LdbBaseTest):
     def test_add_dict_text(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
-             "bla": "bla"}
+             "bla": "bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -356,7 +368,8 @@ class SimpleLdb(LdbBaseTest):
 
     def test_add_dict_string_dn(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
-        m = {"dn": "dc=foo6", "bla": b"bla"}
+        m = {"dn": "dc=foo6", "bla": b"bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -366,7 +379,8 @@ class SimpleLdb(LdbBaseTest):
 
     def test_add_dict_bytes_dn(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
-        m = {"dn": b"dc=foo6", "bla": b"bla"}
+        m = {"dn": b"dc=foo6", "bla": b"bla",
+              "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -379,6 +393,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo7")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -392,6 +407,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo8")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         self.assertEqual(len(l.search()), 1)
@@ -406,14 +422,17 @@ class SimpleLdb(LdbBaseTest):
         self.assertEqual(0, len(l.search()))
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=empty")
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search()
         self.assertEqual(1, len(rm))
-        self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+        self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                         set(rm[0].keys()))
 
         rm = l.search(m.dn)
         self.assertEqual(1, len(rm))
-        self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+        self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                         set(rm[0].keys()))
         rm = l.search(m.dn, attrs=["blah"])
         self.assertEqual(1, len(rm))
         self.assertEqual(0, len(rm[0]))
@@ -423,6 +442,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search(m.dn)[0]
         self.assertEqual([b"1234"], list(rm["bla"]))
@@ -434,7 +454,8 @@ class SimpleLdb(LdbBaseTest):
             l.modify(m)
             rm = l.search(m.dn)
             self.assertEqual(1, len(rm))
-            self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+            self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                             set(rm[0].keys()))
             rm = l.search(m.dn, attrs=["bla"])
             self.assertEqual(1, len(rm))
             self.assertEqual(0, len(rm[0]))
@@ -446,6 +467,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search(m.dn)[0]
         self.assertEqual(["1234"], list(rm.text["bla"]))
@@ -457,7 +479,8 @@ class SimpleLdb(LdbBaseTest):
             l.modify(m)
             rm = l.search(m.dn)
             self.assertEqual(1, len(rm))
-            self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+            self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                             set(rm[0].keys()))
             rm = l.search(m.dn, attrs=["bla"])
             self.assertEqual(1, len(rm))
             self.assertEqual(0, len(rm[0]))
@@ -469,6 +492,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -477,7 +501,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"1234", b"456"], list(rm["bla"]))
         finally:
             l.delete(ldb.Dn(l, "dc=add"))
@@ -487,6 +511,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -495,7 +520,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["1234", "456"], list(rm.text["bla"]))
         finally:
             l.delete(ldb.Dn(l, "dc=add"))
@@ -505,6 +530,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m["bla"] = [b"1234", b"456"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -513,7 +539,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"789"], list(rm["bla"]))
             rm = l.search(m.dn, attrs=["bla"])[0]
             self.assertEqual(1, len(rm))
@@ -525,6 +551,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m.text["bla"] = ["1234", "456"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -533,7 +560,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["789"], list(rm.text["bla"]))
             rm = l.search(m.dn, attrs=["bla"])[0]
             self.assertEqual(1, len(rm))
@@ -545,6 +572,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -553,7 +581,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"1234", b"456"], list(rm["bla"]))
 
             # Now create another modify, but switch the flags before we do it
@@ -571,6 +599,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -579,7 +608,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["1234", "456"], list(rm.text["bla"]))
 
             # Now create another modify, but switch the flags before we do it
@@ -597,6 +626,7 @@ class SimpleLdb(LdbBaseTest):
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo9"))
         m["foo"] = [b"bar"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         l.transaction_commit()
         l.delete(m.dn)
@@ -606,6 +636,7 @@ class SimpleLdb(LdbBaseTest):
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo10"))
         m["foo"] = [b"bar"]
+        m["objectUUID"] = b"0123456789abcdee"
         l.add(m)
         l.transaction_cancel()
         self.assertEqual(0, len(l.search(ldb.Dn(l, "dc=foo10"))))
@@ -625,6 +656,7 @@ class SimpleLdb(LdbBaseTest):
             "cN" : b"LDAPtestUSER",
             "givenname" : b"ldap",
             "displayname" : b"foo\0bar",
+            "objectUUID" : b"0123456789abcdef"
         })
         res = l.search(expression="(dn=dc=somedn)")
         self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
@@ -638,6 +670,10 @@ class SimpleLdbLmdb(SimpleLdb):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXONE": [b"1"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(SimpleLdbLmdb, self).setUp()
 
     def tearDown(self):
@@ -659,6 +695,10 @@ class SearchTests(LdbBaseTest):
         self.l = ldb.Ldb(self.url(),
                          flags=self.flags(),
                          options=["modules:rdn_name"])
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
 
         self.l.add({"dn": "@ATTRIBUTES",
                     "DC": "CASE_INSENSITIVE"})
@@ -670,7 +710,7 @@ class SearchTests(LdbBaseTest):
 
         self.l.add({"dn": "DC=SAMBA,DC=ORG",
                     "name": b"samba.org",
-                    "objectUUID": b"0123456789abcddf"})
+                    "objectUUID": b"0123456789abcdef"})
         self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
                     "name": b"Admins",
                     "x": "z", "y": "a",
@@ -1072,6 +1112,9 @@ class SearchTestsLmdb(SearchTests):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(SearchTestsLmdb, self).setUp()
 
     def tearDown(self):
@@ -1129,12 +1172,12 @@ class GUIDIndexedSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedSearchTests, self).setUp()
 
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDXGUID = True
         self.IDXONE = True
 
@@ -1143,15 +1186,14 @@ class GUIDIndexedDNFilterSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedDNFilterSearchTests, self).setUp()
         self.l.add({"dn": "@OPTIONS",
                     "disallowDNFilter": "TRUE"})
         self.disallowDNFilter = True
-
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDX = True
         self.IDXGUID = True
 
@@ -1159,16 +1201,14 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
     """Test searches using the index including @IDXONE, to ensure
        the index doesn't break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDAndOneLevelIndexedSearchTests, self).setUp()
         self.l.add({"dn": "@OPTIONS",
                     "disallowDNFilter": "TRUE"})
         self.disallowDNFilter = True
-
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXONE": [b"1"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDX = True
         self.IDXGUID = True
         self.IDXONE = True
@@ -1218,6 +1258,11 @@ class AddModifyTests(LdbBaseTest):
         self.l = ldb.Ldb(self.url(),
                          flags=self.flags(),
                          options=["modules:rdn_name"])
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
+
         self.l.add({"dn": "DC=SAMBA,DC=ORG",
                     "name": b"samba.org",
                     "objectUUID": b"0123456789abcdef"})
@@ -1346,23 +1391,15 @@ class AddModifyTests(LdbBaseTest):
                     "objectUUID": b"0123456789abcde3"})
 
 
-class AddModifyTestsLmdb(AddModifyTests):
-
-    def setUp(self):
-        self.prefix = MDB_PREFIX
-        super(AddModifyTestsLmdb, self).setUp()
-
-    def tearDown(self):
-        super(AddModifyTestsLmdb, self).tearDown()
-
 class IndexedAddModifyTests(AddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        if not hasattr(self, 'index'):
+            self.index = {"dn": "@INDEXLIST",
+                          "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
+                          "@IDXONE": [b"1"]}
         super(IndexedAddModifyTests, self).setUp()
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
-                    "@IDXONE": [b"1"]})
 
     def test_duplicate_GUID(self):
         try:
@@ -1436,14 +1473,12 @@ class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXONE": [b"1"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedAddModifyTests, self).setUp()
-        indexlist = {"dn": "@INDEXLIST",
-                     "@IDXATTR": [b"x", b"y", b"ou"],
-                     "@IDXONE": [b"1"],
-                     "@IDXGUID": [b"objectUUID"],
-                     "@IDX_DN_GUID": [b"GUID"]}
-        m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
-        self.l.modify(m)
 
 
 class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
@@ -2260,19 +2295,36 @@ class LdbResultTests(LdbBaseTest):
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
         self.l = ldb.Ldb(self.url(), flags=self.flags())
-        self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
-        self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
-        self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
-        self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1"})
-        self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2"})
-        self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3"})
-        self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4"})
-        self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5"})
-        self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6"})
-        self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7"})
-        self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8"})
-        self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9"})
-        self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10"})
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
+        self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org",
+                    "objectUUID": b"0123456789abcde0"})
+        self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins",
+                    "objectUUID": b"0123456789abcde1"})
+        self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users",
+                    "objectUUID": b"0123456789abcde2"})
+        self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1",
+                    "objectUUID": b"0123456789abcde3"})
+        self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2",
+                    "objectUUID": b"0123456789abcde4"})
+        self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3",
+                    "objectUUID": b"0123456789abcde5"})
+        self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4",
+                    "objectUUID": b"0123456789abcde6"})
+        self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5",
+                    "objectUUID": b"0123456789abcde7"})
+        self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6",
+                    "objectUUID": b"0123456789abcde8"})
+        self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7",
+                    "objectUUID": b"0123456789abcde9"})
+        self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8",
+                    "objectUUID": b"0123456789abcdea"})
+        self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9",
+                    "objectUUID": b"0123456789abcdeb"})
+        self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10",
+                    "objectUUID": b"0123456789abcdec"})
 
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -2373,7 +2425,8 @@ class LdbResultTests(LdbBaseTest):
 
             # write to it
             child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
-                           "name": b"samba.org"})
+                           "name": b"samba.org",
+                           "objectUUID": b"o123456789acbdef"})
 
             os.write(w1, b"added")
 
@@ -2444,7 +2497,8 @@ class LdbResultTests(LdbBaseTest):
 
             # write to it
             child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
-                           "name": b"samba.org"})
+                           "name": b"samba.org",
+                           "objectUUID": b"o123456789acbdef"})
 
             os.write(w1, b"added")
 
@@ -2508,6 +2562,9 @@ class LdbResultTestsLmdb(LdbResultTests):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(LdbResultTestsLmdb, self).setUp()
 
     def tearDown(self):
-- 
2.11.0


From 7db22499e0d60fac2e3b0b2fdc0b4c2dc802cf96 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 40/54] ldb_mdb: Apply LMDB key length restrictions at
 key-value layer

We need to enforce the GUID index mode so end-users do not get a supprise
in mid-operation and we enforce a max key length of 511 so that the
index key trunctation is done correctly.

Otherwise the DB will appear to work until a very long key (DN or index)
is used, after which it will be sad.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c     |  22 +++++-
 lib/ldb/ldb_tdb/ldb_tdb.c     |  17 ++++-
 lib/ldb/tests/ldb_lmdb_test.c | 166 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 200 insertions(+), 5 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 711700c2697..f05d1201e05 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -30,6 +30,8 @@
 #define MDB_URL_PREFIX		"mdb://"
 #define MDB_URL_PREFIX_SIZE	(sizeof(MDB_URL_PREFIX)-1)
 
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
 #define MEGABYTE (1024*1024)
 #define GIGABYTE (1024*1024*1024)
 
@@ -641,7 +643,8 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 {
 	int ret;
 	unsigned int mdb_flags;
-
+	int lmdb_max_key_length;
+	
 	if (flags & LDB_FLG_DONT_CREATE_DB) {
 		struct stat st;
 		if (stat(path, &st) != 0) {
@@ -695,6 +698,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 	/* Store the original pid during the LMDB open */
 	lmdb->pid = getpid();
 
+	lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+	/* This will never happen, but if it does make sure to freak out */
+	if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+		talloc_free(lmdb);
+		return ldb_operr(ldb);
+	}
+
 	return LDB_SUCCESS;
 
 }
@@ -746,6 +757,15 @@ int lmdb_connect(struct ldb_context *ldb,
 	if (flags & LDB_FLG_RDONLY) {
 		ltdb->read_only = true;
 	}
+
+	/*
+	 * This maximum length becomes encoded in the index values so
+	 * must never change even if LMDB starts to allow longer keys.
+	 * The override option is max_key_len_for_self_test, and is
+	 * used for testing only.
+	 */
+	ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+	
         return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
 }
 
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1d06566aa30..b1158241ef9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -651,6 +651,15 @@ static int ltdb_add(struct ltdb_context *ctx)
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	int ret = LDB_SUCCESS;
 
+	if (ltdb->max_key_length != 0 && 
+	    ltdb->cache->GUID_index_attribute == NULL &&
+	    !ldb_dn_is_special(req->op.add.message->dn)) {
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				  "Must operate ldb_mdb in GUID "
+				  "index mode, but " LTDB_IDXGUID " not set.");
+			return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+	
 	ret = ltdb_check_special_dn(module, req->op.add.message);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -2149,7 +2158,13 @@ int init_store(struct ltdb_private *ltdb,
 
 	*_module = module;
 	/*
-	 * Set the maximum key length
+	 * Set or override the maximum key length
+	 *
+	 * The ldb_mdb code will have set this to 511, but our tests
+	 * set this even smaller (to make the tests more practical).
+	 *
+	 * This must only be used for the selftest as the length
+	 * becomes encoded in the index keys.
 	 */
 	{
 		const char *len_str =
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index 2902eaf0533..fd82867a275 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -135,12 +135,22 @@ static int ldbtest_setup(void **state)
 {
 	struct ldbtest_ctx *test_ctx;
 	int ret;
+	struct ldb_ldif *ldif;
+	const char *index_ldif =		\
+		"dn: @INDEXLIST\n"
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+		"\n";
 
 	ldbtest_noconn_setup((void **) &test_ctx);
 
 	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
 	assert_int_equal(ret, 0);
 
+	while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+		ret = ldb_add(test_ctx->ldb, ldif->msg);
+		assert_int_equal(ret, LDB_SUCCESS);
+	}
 	*state = test_ctx;
 	return 0;
 }
@@ -170,7 +180,8 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	assert_non_null(msg);
 
 	/*
-	 * The zero terminator is part of the key
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
 	 */
 
 	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
@@ -184,8 +195,11 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
-	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+	assert_int_equal(ret, LDB_SUCCESS);
 
 	talloc_free(tmp_ctx);
 }
@@ -207,7 +221,8 @@ static void test_ldb_add_key_len_eq_max(void **state)
 	assert_non_null(msg);
 
 	/*
-	 * The zero terminator is part of the key
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
 	 */
 
 	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
@@ -220,12 +235,145 @@ static void test_ldb_add_key_len_eq_max(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
 	talloc_free(tmp_ctx);
 }
 
+static int ldbtest_setup_noguid(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static void test_ldb_add_special_key_len_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 5;  /* "dn=@" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_special_key_len_eq_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 5;  /* "dn=@" and the zero terminator */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_SUCCESS);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_dn_no_guid_mode(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
+
+	talloc_free(tmp_ctx);
+}
+
 int main(int argc, const char **argv)
 {
 	const struct CMUnitTest tests[] = {
@@ -237,6 +385,18 @@ int main(int argc, const char **argv)
 			test_ldb_add_key_len_gt_max,
 			ldbtest_setup,
 			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_special_key_len_eq_max,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_special_key_len_gt_max,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_dn_no_guid_mode,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);
-- 
2.11.0


From 92a33d399bee0db93b9efac2e392a68e60256c97 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 41/54] ldb_mdb: Wrap mdb_env_open

Wrap mdb_env_open to ensure that we only have one MDB_env opened per
database in each process

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 155 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 121 insertions(+), 34 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f05d1201e05..4ab3b3f26d9 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -157,6 +157,9 @@ static int lmdb_store(struct ltdb_private *ltdb,
 	MDB_txn *txn = NULL;
 	MDB_dbi dbi = 0;
 
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
 	if (txn == NULL) {
 		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -212,6 +215,10 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
 	MDB_txn *txn = NULL;
 	MDB_dbi dbi = 0;
 
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
 	if (txn == NULL) {
 		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -628,73 +635,154 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 		trans_finished(lmdb, ltx);
 		ltx = lmdb_private_trans_head(lmdb);
 	}
-
-	mdb_env_close(lmdb->env);
 	lmdb->env = NULL;
 
 	return 0;
 }
 
-static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
-				  struct ldb_context *ldb,
-				  const char *path,
-				  unsigned int flags,
-				  struct lmdb_private *lmdb)
+struct mdb_env_wrap {
+	struct mdb_env_wrap *next, *prev;
+	dev_t device;
+	ino_t inode;
+	MDB_env *env;
+	int pid;
+};
+
+static struct mdb_env_wrap *mdb_list;
+
+/* destroy the last connection to an mdb */
+static int mdb_env_wrap_destructor(struct mdb_env_wrap *w)
 {
+	mdb_env_close(w->env);
+	DLIST_REMOVE(mdb_list, w);
+	return 0;
+}
+
+static int lmdb_open_env(TALLOC_CTX *mem_ctx,
+			 MDB_env **env,
+			 struct ldb_context *ldb,
+			 const char *path,
+			 unsigned int flags) {
 	int ret;
-	unsigned int mdb_flags;
-	int lmdb_max_key_length;
-	
-	if (flags & LDB_FLG_DONT_CREATE_DB) {
-		struct stat st;
-		if (stat(path, &st) != 0) {
-			return LDB_ERR_UNAVAILABLE;
+	unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+	/*
+	 * MDB_NOSUBDIR implies there is a separate file called path and a
+	 * separate lockfile called path-lock
+	 */
+
+	struct mdb_env_wrap *w;
+	struct stat st;
+
+	if (stat(path, &st) == 0) {
+		for (w=mdb_list;w;w=w->next) {
+			if (st.st_dev == w->device && st.st_ino == w->inode) {
+				/*
+				 * We must have only one MDB_env per process
+				 */
+				if (!talloc_reference(mem_ctx, w)) {
+					return ldb_oom(ldb);
+				}
+				*env = w->env;
+				return LDB_SUCCESS;
+			}
 		}
 	}
 
-	ret = mdb_env_create(&lmdb->env);
+	w = talloc(mem_ctx, struct mdb_env_wrap);
+	if (w == NULL) {
+		return ldb_oom(ldb);
+	}
+
+	ret = mdb_env_create(env);
 	if (ret != 0) {
 		ldb_asprintf_errstring(
 			ldb,
 			"Could not create MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
-		return LDB_ERR_OPERATIONS_ERROR;
+		return ldb_mdb_err_map(ret);
 	}
 
-	/* Close when lmdb is released */
-	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
-
-	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+	/*
+	 * Currently we set a 16Gb maximum database size
+	 */
+	ret = mdb_env_set_mapsize(*env, 16LL * GIGABYTE);
 	if (ret != 0) {
 		ldb_asprintf_errstring(
 			ldb,
 			"Could not open MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
+		TALLOC_FREE(w);
 		return ldb_mdb_err_map(ret);
 	}
 
-	mdb_env_set_maxreaders(lmdb->env, 100000);
-	/* MDB_NOSUBDIR implies there is a separate file called path and a
-	 * separate lockfile called path-lock
+	mdb_env_set_maxreaders(*env, 100000);
+	/*
+	 * As we ensure that there is only one MDB_env open per database per
+	 * process. We can not use the MDB_RDONLY flag, as another ldb may be
+	 * opened in read write mode
 	 */
-	mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
-	if (flags & LDB_FLG_RDONLY) {
-		mdb_flags |= MDB_RDONLY;
-	}
 	if (flags & LDB_FLG_NOSYNC) {
 		mdb_flags |= MDB_NOSYNC;
 	}
-	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+	ret = mdb_env_open(*env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
 				"Could not open DB %s: %s\n",
 				path, mdb_strerror(ret));
-		talloc_free(lmdb);
+		TALLOC_FREE(w);
 		return ldb_mdb_err_map(ret);
 	}
 
+	if (stat(path, &st) != 0) {
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not stat %s:\n",
+			path);
+		TALLOC_FREE(w);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	w->env = *env;
+	w->device = st.st_dev;
+	w->inode  = st.st_ino;
+
+	talloc_set_destructor(w, mdb_env_wrap_destructor);
+
+	DLIST_ADD(mdb_list, w);
+
+	return LDB_SUCCESS;
+
+}
+
+static int lmdb_pvt_open(struct lmdb_private *lmdb,
+			 struct ldb_context *ldb,
+			 const char *path,
+			 unsigned int flags)
+{
+	int ret;
+	int lmdb_max_key_length;
+
+	if (flags & LDB_FLG_DONT_CREATE_DB) {
+		struct stat st;
+		if (stat(path, &st) != 0) {
+			return LDB_ERR_UNAVAILABLE;
+		}
+	}
+
+	ret = lmdb_open_env(lmdb, &lmdb->env, ldb, path, flags);
+	if (ret != 0) {
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not create MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	/* Close when lmdb is released */
+	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
 	/* Store the original pid during the LMDB open */
 	lmdb->pid = getpid();
 
@@ -702,7 +790,6 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 
 	/* This will never happen, but if it does make sure to freak out */
 	if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
-		talloc_free(lmdb);
 		return ldb_operr(ldb);
 	}
 
@@ -739,17 +826,17 @@ int lmdb_connect(struct ldb_context *ldb,
                 return LDB_ERR_OPERATIONS_ERROR;
         }
 
-	lmdb = talloc_zero(ldb, struct lmdb_private);
+	lmdb = talloc_zero(ltdb, struct lmdb_private);
 	if (lmdb == NULL) {
 		TALLOC_FREE(ltdb);
-                ldb_oom(ldb);
-                return LDB_ERR_OPERATIONS_ERROR;
+                return ldb_oom(ldb);
 	}
 	lmdb->ldb = ldb;
 	ltdb->kv_ops = &lmdb_key_value_ops;
 
-	ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+	ret = lmdb_pvt_open(lmdb, ldb, path, flags);
 	if (ret != LDB_SUCCESS) {
+		TALLOC_FREE(ltdb);
 		return ret;
 	}
 
-- 
2.11.0


From 13d4e31f946b59fc31fd25a7d8b9e5a8abaaece1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 8 Mar 2018 16:47:59 +1300
Subject: [PATCH 42/54] ldb_mdb: Tests for wrap open

Tests to ensure that the mdb_env wrapping code correctly handles
multiple ldb's point to the same physical database file.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 234 ++++++++++++++++++++++++++++++++++++++++
 lib/ldb/wscript                 |   5 +-
 2 files changed, 237 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 6582f37f7d2..1fce9f97882 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -38,6 +38,12 @@
 #define TEST_BE DEFAULT_BE
 #endif /* TEST_BE */
 
+#ifdef TEST_LMDB
+#include "lmdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+#endif
+
 struct ldbtest_ctx {
 	struct tevent_context *ev;
 	struct ldb_context *ldb;
@@ -3820,6 +3826,224 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
 	}
 }
 
+#ifdef TEST_LMDB
+static int test_ldb_multiple_connections_callback(struct ldb_request *req,
+						  struct ldb_reply *ares)
+{
+	int ret;
+	int pipes[2];
+	char buf[2];
+	int pid, child_pid;
+	int wstatus;
+
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
+		break;
+
+	case LDB_REPLY_REFERRAL:
+		return LDB_SUCCESS;
+
+	case LDB_REPLY_DONE:
+		return ldb_request_done(req, LDB_SUCCESS);
+	}
+
+	{
+		/*
+		 * We open a new ldb on an ldb that is already open and
+		 * then close it.
+		 *
+		 * If the multiple connection wrapping is correct the
+		 * underlying MDB_env will be left open and we should see
+		 * an active reader in the child we fork next
+		 */
+		struct ldb_context *ldb = NULL;
+		struct tevent_context *ev = NULL;
+		TALLOC_CTX *mem_ctx = talloc_new(NULL);
+
+		ev = tevent_context_init(mem_ctx);
+		assert_non_null(ev);
+
+		ldb = ldb_init(mem_ctx, ev);
+		assert_non_null(ldb);
+
+		ret = ldb_connect(ldb, TEST_BE"://apitest.ldb" , 0, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		TALLOC_FREE(ldb);
+		TALLOC_FREE(mem_ctx);
+	}
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct MDB_env *env = NULL;
+		struct MDB_envinfo stat;
+
+		/*
+		 * Check that there are exactly two readers on the MDB file
+		 * backing the ldb.
+		 *
+		 */
+		ret = mdb_env_create(&env);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_create returned (%d)",
+				      ret);
+			exit(ret);
+		}
+
+		ret = mdb_env_open(env,
+				   "apitest.ldb",
+				   MDB_NOSUBDIR | MDB_NOTLS,
+				   0644);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_open returned (%d)",
+				      ret);
+			exit(ret);
+		}
+
+		ret = mdb_env_info(env, &stat);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_info returned (%d)",
+				      ret);
+			exit(ret);
+		}
+		if (stat.me_numreaders != 2) {
+			print_error(__location__
+				      " Incorrect number of readers (%d)",
+				      stat.me_numreaders);
+			exit(LDB_ERR_CONSTRAINT_VIOLATION);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+	return LDB_SUCCESS;
+
+}
+
+static void test_ldb_close_with_multiple_connections(void **state)
+{
+	struct search_test_ctx *search_test_ctx = NULL;
+	struct ldb_dn *search_dn = NULL;
+	struct ldb_request *req = NULL;
+	int ret = 0;
+
+	search_test_ctx = talloc_get_type_abort(*state, struct search_test_ctx);
+	assert_non_null(search_test_ctx);
+
+	search_dn = ldb_dn_new_fmt(search_test_ctx,
+				   search_test_ctx->ldb_test_ctx->ldb,
+				   "cn=test_search_cn,"
+				   "dc=search_test_entry");
+	assert_non_null(search_dn);
+
+	/*
+	 * The search just needs to call DONE, we don't care about the
+	 * contents of the search for this test
+	 */
+	ret = ldb_build_search_req(&req,
+				   search_test_ctx->ldb_test_ctx->ldb,
+				   search_test_ctx,
+				   search_dn,
+				   LDB_SCOPE_SUBTREE,
+				   "(&(!(filterAttr=*))"
+				   "(cn=test_search_cn))",
+				   NULL,
+				   NULL,
+				   NULL,
+				   test_ldb_multiple_connections_callback,
+				   NULL);
+	assert_int_equal(ret, 0);
+
+	ret = ldb_request(search_test_ctx->ldb_test_ctx->ldb, req);
+	assert_int_equal(ret, 0);
+
+	ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	assert_int_equal(ret, 0);
+}
+
+static struct MDB_env* get_mdb_env(struct ldb_context *ldb)
+{
+	void *data = NULL;
+	struct ltdb_private *ltdb = NULL;
+	struct lmdb_private *lmdb = NULL;
+	struct MDB_env *env = NULL;
+
+	data = ldb_module_get_private(ldb->modules);
+	assert_non_null(data);
+
+	ltdb = talloc_get_type(data, struct ltdb_private);
+	assert_non_null(ltdb);
+
+	lmdb = ltdb->lmdb_private;
+	assert_non_null(lmdb);
+
+	env = lmdb->env;
+	assert_non_null(env);
+
+	return env;
+}
+static void test_multiple_opens(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	struct ldb_context *ldb2 = NULL;
+	struct ldb_context *ldb3 = NULL;
+	struct MDB_env *env1 = NULL;
+	struct MDB_env *env2 = NULL;
+	struct MDB_env *env3 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database again
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb2 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb3 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+	/*
+	 * We now have 3 ldb's open pointing to the same on disk database
+	 * they should all share the same MDB_env
+	 */
+	env1 = get_mdb_env(ldb1);
+	env2 = get_mdb_env(ldb2);
+	env3 = get_mdb_env(ldb3);
+
+	assert_ptr_equal(env1, env2);
+	assert_ptr_equal(env1, env3);
+
+}
+#endif
 
 int main(int argc, const char **argv)
 {
@@ -3984,6 +4208,16 @@ int main(int argc, const char **argv)
 			test_ldb_talloc_destructor_transaction_cleanup,
 			ldbtest_setup,
 			ldbtest_teardown),
+#ifdef TEST_LMDB
+		cmocka_unit_test_setup_teardown(
+			test_ldb_close_with_multiple_connections,
+			ldb_search_test_setup,
+			ldb_search_test_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_multiple_opens,
+			ldbtest_setup,
+			ldbtest_teardown),
+#endif
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index d9f47c798bf..3da73c16d7c 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -443,8 +443,9 @@ def build(bld):
         if bld.CONFIG_SET('HAVE_LMDB'):
             bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
                              source='tests/ldb_mod_op_test.c',
-                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
-                             deps='cmocka ldb',
+                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 ' +
+                                    '-DTEST_LMDB=1',
+                             deps='cmocka ldb lmdb',
                              install=False)
             
             bld.SAMBA_BINARY('ldb_lmdb_test',
-- 
2.11.0


From 6b50f5e763444c2917d3d017cb2250cf5afd286d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:14:09 +1300
Subject: [PATCH 43/54] ldb index tests: test large index key value

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_lmdb_test.c | 44 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index fd82867a275..92715fdf96f 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -204,6 +204,46 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	talloc_free(tmp_ctx);
 }
 
+static void test_ldb_add_key_len_2x_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = 2 * LMDB_MAX_KEY_SIZE;
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_SUCCESS);
+
+	talloc_free(tmp_ctx);
+}
+
 static void test_ldb_add_key_len_eq_max(void **state)
 {
 	int ret;
@@ -386,6 +426,10 @@ int main(int argc, const char **argv)
 			ldbtest_setup,
 			ldbtest_teardown),
 		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_2x_gt_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
 			test_ldb_add_special_key_len_eq_max,
 			ldbtest_setup_noguid,
 			ldbtest_teardown),
-- 
2.11.0


From 152b3024e737718693f912ed669e948ecc48e51c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 12:43:25 +1300
Subject: [PATCH 44/54] ldb test: close pipes to stop forked tests failing on
 failure

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 1fce9f97882..c33517134df 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3881,6 +3881,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
 	if (child_pid == 0) {
 		struct MDB_env *env = NULL;
 		struct MDB_envinfo stat;
+		close(pipes[0]);
 
 		/*
 		 * Check that there are exactly two readers on the MDB file
@@ -3929,6 +3930,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
 		}
 		exit(LDB_SUCCESS);
 	}
+	close(pipes[1]);
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
 
-- 
2.11.0


From 5271ebf75079bfbb9136bf34142b4d195ed7ec4a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 45/54] ldb_mdb: prevent MDB_env reuse across forks

MDB_env's may not be reused accross forks.  Check the pid that the lmdb
structure was created by, and return an error if it is being used by a
different process.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c       |  33 +++++-
 lib/ldb/tests/ldb_mod_op_test.c | 233 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 264 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 4ab3b3f26d9..eb4609ee98e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -433,6 +433,18 @@ static int lmdb_lock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	pid_t pid = getpid();
+
+	if (pid != lmdb->pid) {
+		ldb_asprintf_errstring(
+			lmdb->ldb,
+			__location__": Reusing ldb opend by pid %d in "
+			"process %d\n",
+			lmdb->pid,
+			pid);
+		lmdb->error = MDB_BAD_TXN;
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
 
 	lmdb->error = MDB_SUCCESS;
 	if (ltdb->in_transaction == 0 &&
@@ -472,12 +484,25 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 	struct lmdb_trans *ltx;
 	struct lmdb_trans *ltx_head;
 	MDB_txn *tx_parent;
+	pid_t pid = getpid();
 
 	ltx = talloc_zero(lmdb, struct lmdb_trans);
 	if (ltx == NULL) {
 		return ldb_oom(lmdb->ldb);
 	}
 
+	if (pid != lmdb->pid) {
+		ldb_asprintf_errstring(
+			lmdb->ldb,
+			__location__": Reusing ldb opened by pid %d in "
+			"process %d\n",
+			lmdb->pid,
+			pid);
+		lmdb->error = MDB_BAD_TXN;
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
+
+
 	ltx_head = lmdb_private_trans_head(lmdb);
 
 	tx_parent = lmdb_trans_get_tx(ltx_head);
@@ -645,7 +670,7 @@ struct mdb_env_wrap {
 	dev_t device;
 	ino_t inode;
 	MDB_env *env;
-	int pid;
+	pid_t pid;
 };
 
 static struct mdb_env_wrap *mdb_list;
@@ -672,10 +697,13 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
 
 	struct mdb_env_wrap *w;
 	struct stat st;
+	pid_t pid = getpid();
 
 	if (stat(path, &st) == 0) {
 		for (w=mdb_list;w;w=w->next) {
-			if (st.st_dev == w->device && st.st_ino == w->inode) {
+			if (st.st_dev == w->device &&
+			    st.st_ino == w->inode &&
+			    pid == w->pid) {
 				/*
 				 * We must have only one MDB_env per process
 				 */
@@ -746,6 +774,7 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
 	w->env = *env;
 	w->device = st.st_dev;
 	w->inode  = st.st_ino;
+	w->pid = pid;
 
 	talloc_set_destructor(w, mdb_env_wrap_destructor);
 
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index c33517134df..da20485e016 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -4045,6 +4045,227 @@ static void test_multiple_opens(void **state)
 	assert_ptr_equal(env1, env3);
 
 }
+
+static void test_multiple_opens_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	struct ldb_context *ldb2 = NULL;
+	struct MDB_env *env1 = NULL;
+	struct MDB_env *env2 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database again
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb2 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	env1 = get_mdb_env(ldb1);
+	env2 = get_mdb_env(ldb2);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct ldb_context *ldb3 = NULL;
+		struct MDB_env *env3 = NULL;
+
+		close(pipes[0]);
+		ldb3 = ldb_init(test_ctx, test_ctx->ev);
+		ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+		if (ret != 0) {
+			print_error(__location__": ldb_connect returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+		env3 = get_mdb_env(ldb3);
+		if (env1 != env2) {
+			print_error(__location__": env1 != env2\n");
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (env1 == env3) {
+			print_error(__location__": env1 == env3\n");
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_transaction_start_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		close(pipes[0]);
+		ret = ldb_transaction_start(ldb1);
+		if (ret != LDB_ERR_PROTOCOL_ERROR) {
+			print_error(__location__": ldb_transaction_start "
+				    "returned (%d) %s\n",
+				    ret,
+				    ldb1->err_string);
+			exit(LDB_ERR_OTHER);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_lock_read_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct ldb_dn *basedn;
+		struct ldb_result *result = NULL;
+
+		close(pipes[0]);
+
+		basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+		assert_non_null(basedn);
+
+		ret = ldb_search(test_ctx->ldb,
+				 test_ctx,
+				 &result,
+				 basedn,
+				 LDB_SCOPE_BASE,
+				 NULL,
+				 NULL);
+		if (ret != LDB_ERR_PROTOCOL_ERROR) {
+			print_error(__location__": ldb_search "
+				    "returned (%d) %s\n",
+				    ret,
+				    ldb1->err_string);
+			exit(LDB_ERR_OTHER);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+	{
+		/*
+		 * Ensure that the search actually succeeds on the opening
+		 * pid
+		 */
+		struct ldb_dn *basedn;
+		struct ldb_result *result = NULL;
+
+		close(pipes[0]);
+
+		basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+		assert_non_null(basedn);
+
+		ret = ldb_search(test_ctx->ldb,
+				 test_ctx,
+				 &result,
+				 basedn,
+				 LDB_SCOPE_BASE,
+				 NULL,
+				 NULL);
+		assert_int_equal(0, ret);
+	}
+}
 #endif
 
 int main(int argc, const char **argv)
@@ -4219,6 +4440,18 @@ int main(int argc, const char **argv)
 			test_multiple_opens,
 			ldbtest_setup,
 			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_multiple_opens_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_transaction_start_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_lock_read_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
 #endif
 	};
 
-- 
2.11.0


From acb7db2c25988d2405095bf8738588efaf32647a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:25:28 +1300
Subject: [PATCH 46/54] ldb: make backends expose if there is an active
 transaction

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 37 +++++++++++++++++++++----------------
 lib/ldb/ldb_tdb/ldb_tdb.c |  5 +++++
 lib/ldb/ldb_tdb/ldb_tdb.h |  1 +
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index eb4609ee98e..bb398ba89d5 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -91,6 +91,11 @@ static int lmdb_error_at(struct ldb_context *ldb,
 	return ldb_err;
 }
 
+
+static bool lmdb_transaction_active(struct ltdb_private *ltdb) {
+	return ltdb->lmdb_private->txlist != NULL;
+}
+
 static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
 {
 	if (ltx == NULL) {
@@ -581,23 +586,23 @@ static bool lmdb_changed(struct ltdb_private *ltdb)
 	return true;
 }
 
-
 static struct kv_db_ops lmdb_key_value_ops = {
-	.store             = lmdb_store,
-	.delete            = lmdb_delete,
-	.iterate           = lmdb_traverse_fn,
-	.update_in_iterate = lmdb_update_in_iterate,
-	.fetch_and_parse   = lmdb_parse_record,
-	.lock_read         = lmdb_lock_read,
-	.unlock_read       = lmdb_unlock_read,
-	.begin_write       = lmdb_transaction_start,
-	.prepare_write     = lmdb_transaction_prepare_commit,
-	.finish_write      = lmdb_transaction_commit,
-	.abort_write       = lmdb_transaction_cancel,
-	.error             = lmdb_error,
-	.errorstr          = lmdb_errorstr,
-	.name              = lmdb_name,
-	.has_changed       = lmdb_changed,
+	.store              = lmdb_store,
+	.delete             = lmdb_delete,
+	.iterate            = lmdb_traverse_fn,
+	.update_in_iterate  = lmdb_update_in_iterate,
+	.fetch_and_parse    = lmdb_parse_record,
+	.lock_read          = lmdb_lock_read,
+	.unlock_read        = lmdb_unlock_read,
+	.begin_write        = lmdb_transaction_start,
+	.prepare_write      = lmdb_transaction_prepare_commit,
+	.finish_write       = lmdb_transaction_commit,
+	.abort_write        = lmdb_transaction_cancel,
+	.error              = lmdb_error,
+	.errorstr           = lmdb_errorstr,
+	.name               = lmdb_name,
+	.has_changed        = lmdb_changed,
+	.transaction_active = lmdb_transaction_active,
 };
 
 static const char *lmdb_get_path(const char *url)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index b1158241ef9..66cd6cb9ff7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1924,6 +1924,10 @@ static bool ltdb_tdb_changed(struct ltdb_private *ltdb)
 	return has_changed;
 }
 
+static bool ltdb_transaction_active(struct ltdb_private *ltdb) {
+	return tdb_transaction_active(ltdb->tdb);
+}
+
 static const struct kv_db_ops key_value_ops = {
 	.store = ltdb_tdb_store,
 	.delete = ltdb_tdb_delete,
@@ -1940,6 +1944,7 @@ static const struct kv_db_ops key_value_ops = {
 	.errorstr = ltdb_errorstr,
 	.name = ltdb_tdb_name,
 	.has_changed = ltdb_tdb_changed,
+	.transaction_active = ltdb_transaction_active,
 };
 
 static void ltdb_callback(struct tevent_context *ev,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 7d92804a8e9..59fd06aac40 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -29,6 +29,7 @@ struct kv_db_ops {
 	const char * (*errorstr)(struct ltdb_private *ltdb);
 	const char * (*name)(struct ltdb_private *ltdb);
 	bool (*has_changed)(struct ltdb_private *ltdb);
+	bool (*transaction_active)(struct ltdb_private *ltdb);
 };
 
 /* this private structure is used by the ltdb backend in the
-- 
2.11.0


From e51188693295c1acdb6df272735297a7b7f4be45 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:27:10 +1300
Subject: [PATCH 47/54] ldb_tdb: Do not make search or DB modifications without
 a lock

The ldb_cache startup code would previously not take a read lock
nor a sufficiently wide write transaction.

The new code takes a read lock, and if it needs to write takes a
write lock (transaction) and re-reads before continuing.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_cache.c | 50 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 4790bcd7e53..0d16452b913 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -386,6 +386,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	uint64_t seq;
 	struct ldb_message *baseinfo = NULL, *options = NULL;
 	const struct ldb_schema_attribute *a;
+	bool have_write_txn = false;
 	int r;
 
 	ldb = ldb_module_get_ctx(module);
@@ -406,29 +407,38 @@ int ltdb_cache_load(struct ldb_module *module)
 	baseinfo_dn = ldb_dn_new(baseinfo, ldb, LTDB_BASEINFO);
 	if (baseinfo_dn == NULL) goto failed;
 
+	r = ltdb->kv_ops->lock_read(module);
+	if (r != LDB_SUCCESS) {
+		goto failed;
+	}
 	r= ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0);
 	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
-		goto failed;
+		goto failed_and_unlock;
 	}
-	
+
 	/* possibly initialise the baseinfo */
 	if (r == LDB_ERR_NO_SUCH_OBJECT) {
 
+		/* Give up the read lock, try again with a write lock */
+		r = ltdb->kv_ops->unlock_read(module);
+		if (r != LDB_SUCCESS) {
+			goto failed;
+		}
+		
 		if (ltdb->kv_ops->begin_write(ltdb) != 0) {
 			goto failed;
 		}
 
+		have_write_txn = true;
+		
 		/* error handling for ltdb_baseinfo_init() is by
 		   looking for the record again. */
 		ltdb_baseinfo_init(module);
 
-		if (ltdb->kv_ops->finish_write(ltdb) != 0) {
-			goto failed;
-		}
-
 		if (ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0) != LDB_SUCCESS) {
-			goto failed;
+			goto failed_and_unlock;
 		}
+
 	}
 
 	/* Ignore the result, and update the sequence number */
@@ -443,16 +453,17 @@ int ltdb_cache_load(struct ldb_module *module)
 	ltdb->sequence_number = seq;
 
 	/* Read an interpret database options */
+	
 	options = ldb_msg_new(ltdb->cache);
-	if (options == NULL) goto failed;
+	if (options == NULL) goto failed_and_unlock;
 
 	options_dn = ldb_dn_new(options, ldb, LTDB_OPTIONS);
-	if (options_dn == NULL) goto failed;
+	if (options_dn == NULL) goto failed_and_unlock;
 
 	r= ltdb_search_dn1(module, options_dn, options, 0);
 	talloc_free(options_dn);
 	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 	
 	/* set flags if they do exist */
@@ -479,7 +490,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	ltdb_attributes_unload(module);
 
 	if (ltdb_index_load(module, ltdb) == -1) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 
 	/*
@@ -488,7 +499,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	 * partition module.
 	 */
 	if (ltdb_attributes_load(module) == -1) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 
 	ltdb->GUID_index_syntax = NULL;
@@ -503,10 +514,25 @@ int ltdb_cache_load(struct ldb_module *module)
 	}
 
 done:
+	if (have_write_txn) {
+		if (ltdb->kv_ops->finish_write(ltdb) != 0) {
+			goto failed;
+		}
+	} else {
+		ltdb->kv_ops->unlock_read(module);
+	}
+	
 	talloc_free(options);
 	talloc_free(baseinfo);
 	return 0;
 
+failed_and_unlock:
+	if (have_write_txn) {
+		ltdb->kv_ops->abort_write(ltdb);
+	} else {
+		ltdb->kv_ops->unlock_read(module);
+	}
+	
 failed:
 	talloc_free(options);
 	talloc_free(baseinfo);
-- 
2.11.0


From 109de608e9f5687ceb72911106f0804e2d9a8dc3 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:29:25 +1300
Subject: [PATCH 48/54] ldb_mdb: Remove implicit read lock and remove
 transaction counter

The way to know if we are in a transaction is if there is a non-NULL
transaction handle.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 51 +++++++++++++++--------------------------------
 1 file changed, 16 insertions(+), 35 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index bb398ba89d5..3c2dc980349 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -129,26 +129,21 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
 	return ltx;
 }
 
+
 static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
 {
 	MDB_txn *txn;
-	if (lmdb->read_txn != NULL) {
-		return lmdb->read_txn;
-	}
 
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
-	if (txn == NULL) {
-		int ret;
-		ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
-		if (ret != 0) {
-			lmdb->error = ret;
-			ldb_asprintf_errstring(lmdb->ldb,
-					       "%s failed: %s\n", __FUNCTION__,
-					       mdb_strerror(ret));
-		}
-		lmdb->read_txn = txn;
+	if (txn != NULL) {
+		return txn;
+	}
+	if (lmdb->read_txn != NULL) {
+		return lmdb->read_txn;
 	}
-	return txn;
+	lmdb->error = MDB_BAD_TXN;
+	ldb_set_errstring(lmdb->ldb, __location__":No active transaction\n");
+	return NULL;
 }
 
 static int lmdb_store(struct ltdb_private *ltdb,
@@ -193,10 +188,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
 		MDB_val value;
 		lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
 		if (lmdb->error != MDB_SUCCESS) {
-			if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-				mdb_txn_commit(lmdb->read_txn);
-				lmdb->read_txn = NULL;
-			}
 			return ldb_mdb_error(lmdb->ldb, lmdb->error);
 		}
 		mdb_flags = 0;
@@ -301,11 +292,6 @@ done:
 		mdb_cursor_close(cursor);
 	}
 
-	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-		mdb_txn_commit(lmdb->read_txn);
-		lmdb->read_txn = NULL;
-	}
-
 	if (lmdb->error != MDB_SUCCESS) {
 		return ldb_mdb_error(lmdb->ldb, lmdb->error);
 	}
@@ -409,10 +395,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
 	if (lmdb->error != MDB_SUCCESS) {
 		/* TODO closing a handle should not even be necessary */
 		mdb_dbi_close(lmdb->env, dbi);
-		if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-			mdb_txn_commit(lmdb->read_txn);
-			lmdb->read_txn = NULL;
-		}
 		if (lmdb->error == MDB_NOTFOUND) {
 			return LDB_ERR_NO_SUCH_OBJECT;
 		}
@@ -424,11 +406,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
 	/* TODO closing a handle should not even be necessary */
 	mdb_dbi_close(lmdb->env, dbi);
 
-	/* We created a read transaction, commit it */
-	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-		mdb_txn_commit(lmdb->read_txn);
-		lmdb->read_txn = NULL;
-	}
 	return parser(key, data, ctx);
 }
 
@@ -452,7 +429,7 @@ static int lmdb_lock_read(struct ldb_module *module)
 	}
 
 	lmdb->error = MDB_SUCCESS;
-	if (ltdb->in_transaction == 0 &&
+	if (lmdb_transaction_active(ltdb) == false &&
 	    ltdb->read_lock_count == 0) {
 		lmdb->error = mdb_txn_begin(lmdb->env,
 					    NULL,
@@ -472,7 +449,7 @@ static int lmdb_unlock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+	if (lmdb_transaction_active(ltdb) == false && ltdb->read_lock_count == 1) {
 		struct lmdb_private *lmdb = ltdb->lmdb_private;
 		mdb_txn_commit(lmdb->read_txn);
 		lmdb->read_txn = NULL;
@@ -491,6 +468,11 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 	MDB_txn *tx_parent;
 	pid_t pid = getpid();
 
+	/* Do not take out the transaction lock on a read-only DB */
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+	
 	ltx = talloc_zero(lmdb, struct lmdb_trans);
 	if (ltx == NULL) {
 		return ldb_oom(lmdb->ldb);
@@ -507,7 +489,6 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 		return LDB_ERR_PROTOCOL_ERROR;
 	}
 
-
 	ltx_head = lmdb_private_trans_head(lmdb);
 
 	tx_parent = lmdb_trans_get_tx(ltx_head);
-- 
2.11.0


From e41a651aa74d4a2fa9e5da4d01da0ea336b06b44 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:28:18 +1300
Subject: [PATCH 49/54] ldb_tdb: Disallow TDB nested transactions and use
 tdb_transaction_active()

This avoids keeping a counter, which can be error-prone.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 22 ++++++++++------------
 lib/ldb/ldb_tdb/ldb_tdb.h |  1 -
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 66cd6cb9ff7..babee775008 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -101,7 +101,7 @@ static int ltdb_lock_read(struct ldb_module *module)
 	int tdb_ret = 0;
 	int ret;
 
-	if (ltdb->in_transaction == 0 &&
+	if (tdb_transaction_active(ltdb->tdb) == false &&
 	    ltdb->read_lock_count == 0) {
 		tdb_ret = tdb_lockall_read(ltdb->tdb);
 	}
@@ -128,7 +128,7 @@ static int ltdb_unlock_read(struct ldb_module *module)
 {
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
-	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+	if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
 		tdb_unlockall_read(ltdb->tdb);
 		ltdb->read_lock_count--;
 		return 0;
@@ -379,7 +379,7 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
 
 	/* only allow modifies inside a transaction, otherwise the
 	 * ldb is unsafe */
-	if (ltdb->in_transaction == 0) {
+	if (ltdb->kv_ops->transaction_active(ltdb) == false) {
 		ldb_set_errstring(ldb_module_get_ctx(module), "ltdb modify without transaction");
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -1478,7 +1478,6 @@ static int ltdb_start_trans(struct ldb_module *module)
 		return ltdb->kv_ops->error(ltdb);
 	}
 
-	ltdb->in_transaction++;
 
 	ltdb_index_transaction_start(module);
 
@@ -1499,8 +1498,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	if (ltdb->in_transaction != 1) {
-		return LDB_SUCCESS;
+	if (!ltdb->kv_ops->transaction_active(ltdb)) {
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				  "ltdb_prepare_commit() called "
+				  "without transaction active");
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
 	/*
@@ -1524,13 +1526,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 	ret = ltdb_index_transaction_commit(module);
 	if (ret != LDB_SUCCESS) {
 		ltdb->kv_ops->abort_write(ltdb);
-		ltdb->in_transaction--;
 		return ret;
 	}
 
 	if (ltdb->kv_ops->prepare_write(ltdb) != 0) {
 		ret = ltdb->kv_ops->error(ltdb);
-		ltdb->in_transaction--;
 		ldb_debug_set(ldb_module_get_ctx(module),
 			      LDB_DEBUG_FATAL,
 			      "Failure during "
@@ -1558,7 +1558,6 @@ static int ltdb_end_trans(struct ldb_module *module)
 		}
 	}
 
-	ltdb->in_transaction--;
 	ltdb->prepared_commit = false;
 
 	if (ltdb->kv_ops->finish_write(ltdb) != 0) {
@@ -1578,7 +1577,6 @@ static int ltdb_del_trans(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	ltdb->in_transaction--;
 
 	if (ltdb_index_transaction_cancel(module) != 0) {
 		ltdb->kv_ops->abort_write(ltdb);
@@ -1808,7 +1806,7 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
 		.ctx = ctx,
 		.ltdb = ltdb
 	};
-	if (ltdb->in_transaction != 0) {
+	if (tdb_transaction_active(ltdb->tdb)) {
 		return tdb_traverse(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
 	} else {
 		return tdb_traverse_read(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
@@ -2213,7 +2211,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
 		path = url;
 	}
 
-	tdb_flags = TDB_DEFAULT | TDB_SEQNUM;
+	tdb_flags = TDB_DEFAULT | TDB_SEQNUM | TDB_DISALLOW_NESTING;
 
 	/* check for the 'nosync' option */
 	if (flags & LDB_FLG_NOSYNC) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 59fd06aac40..9c3f8d89d8d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -54,7 +54,6 @@ struct ltdb_private {
 		const char *GUID_index_dn_component;
 	} *cache;
 
-	int in_transaction;
 
 	bool check_base;
 	bool disallow_dn_filter;
-- 
2.11.0


From ec064f732c725b0c775229df17000eb23bb24c03 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 4 Apr 2018 17:21:30 +1200
Subject: [PATCH 50/54] ldb_tdb: Disallow reads without a transaction or read
 lock

This will ensure we match LMDB behaviour and avoid a repeat of the per-record locking
issues (compared with full DB locking) we had before Samba 4.7.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index babee775008..4f31210c5fc 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1899,6 +1899,11 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
 	};
 	int ret;
 
+	if (tdb_transaction_active(ltdb->tdb) == false &&
+	    ltdb->read_lock_count == 0) {
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
+
 	ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
 			       &kv_ctx);
 	if (ret == 0) {
-- 
2.11.0


From 27bed008d1712acb092e05272ea088e34a88b5a9 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:14:10 +1300
Subject: [PATCH 51/54] ldb tests: api ensure database correctly populated

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index bb2c918e9f6..1222d122199 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -1316,6 +1316,19 @@ class AddModifyTests(LdbBaseTest):
                     "name": b"Admins",
                     "x": "z", "y": "a",
                     "objectUUID": b"0123456789abcde2"})
+
+        res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+                             scope=ldb.SCOPE_SUBTREE,
+                             expression="(objectUUID=0123456789abcde1)")
+        self.assertEqual(len(res2), 1)
+        self.assertEqual(str(res2[0].dn), "OU=DUP,DC=SAMBA,DC=ORG")
+
+        res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+                             scope=ldb.SCOPE_SUBTREE,
+                             expression="(objectUUID=0123456789abcde2)")
+        self.assertEqual(len(res3), 1)
+        self.assertEqual(str(res3[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
         try:
             self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
                           "OU=DUP2,DC=SAMBA,DC=ORG")
-- 
2.11.0


From 5d4a9692183f002a9ccf542f65b284d3773b918f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:15:12 +1300
Subject: [PATCH 52/54] ldb tests: add cmocka tests of kv operation
 interactions with transactions

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_kv_ops_test.c | 184 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 184 insertions(+)

diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
index 4cfb618ea94..501d1f1599b 100644
--- a/lib/ldb/tests/ldb_kv_ops_test.c
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -270,6 +270,63 @@ static void test_add_get(void **state)
 }
 
 /*
+ * Test that attempts to read data without a read transaction fail.
+ */
+static void test_read_outside_transaction(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 * Note there is no read transaction active
+	 */
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
  * Test that data can be deleted from the kv store
  */
 static void test_delete(void **state)
@@ -731,6 +788,125 @@ static void test_iterate(void **state)
 	TALLOC_FREE(tmp_ctx);
 }
 
+struct update_context {
+	struct ldb_context* ldb;
+	int visits[NUM_RECS];
+};
+
+static int update_fn(struct ltdb_private *ltdb,
+		     struct ldb_val key,
+		     struct ldb_val data,
+		     void *ctx) {
+
+	struct ldb_val new_key;
+	struct ldb_module *module = NULL;
+	struct update_context *context =NULL;
+	int ret = LDB_SUCCESS;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(ltdb);
+	assert_non_null(tmp_ctx);
+
+	context = talloc_get_type_abort(ctx, struct update_context);
+
+	module = talloc_zero(tmp_ctx, struct ldb_module);
+	module->ldb = context->ldb;
+
+	if (strncmp("key ", (char *) key.data, 4) == 0) {
+		int i = strtol((char *) &key.data[4], NULL, 10);
+		context->visits[i]++;
+		new_key.data = talloc_memdup(tmp_ctx, key.data, key.length);
+		new_key.length  = key.length;
+		new_key.data[0] = 'K';
+
+		ret = ltdb->kv_ops->update_in_iterate(ltdb,
+						      key,
+						      new_key,
+						      data,
+						      &module);
+	}
+	TALLOC_FREE(tmp_ctx);
+	return ret;
+}
+
+/*
+ * Test that update_in_iterate behaves as expected.
+ */
+static void test_update_in_iterate(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	int i;
+	struct update_context *context = NULL;
+
+
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	context = talloc_zero(tmp_ctx, struct update_context);
+	assert_non_null(context);
+	context->ldb = test_ctx->ldb;
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the records
+	 */
+	for (i = 0; i < NUM_RECS; i++) {
+		struct ldb_val key;
+		struct ldb_val rec;
+		int flags = 0;
+
+		key.data   = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+		key.length = strlen((char *)key.data) + 1;
+
+		rec.data   = (uint8_t *) talloc_asprintf(tmp_ctx,
+							 "data for record (%04d)",
+							 i);
+		rec.length = strlen((char *)rec.data) + 1;
+
+		ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+		assert_int_equal(ret, 0);
+
+		TALLOC_FREE(key.data);
+		TALLOC_FREE(rec.data);
+	}
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now iterate over the kv store and ensure that all the
+	 * records are visited.
+	 */
+
+	/*
+	 * Needs to be done inside a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	ret = ltdb->kv_ops->iterate(ltdb, update_fn, context);
+	for (i = 0; i < NUM_RECS; i++) {
+		assert_int_equal(1, context->visits[i]);
+	}
+
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	TALLOC_FREE(tmp_ctx);
+}
+
 /*
  * Ensure that writes are not visible until the transaction has been
  * committed.
@@ -1371,6 +1547,10 @@ int main(int argc, const char **argv)
 			setup,
 			teardown),
 		cmocka_unit_test_setup_teardown(
+			test_read_outside_transaction,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
 			test_write_outside_transaction,
 			setup,
 			teardown),
@@ -1383,6 +1563,10 @@ int main(int argc, const char **argv)
 			setup,
 			teardown),
 		cmocka_unit_test_setup_teardown(
+			test_update_in_iterate,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
 			test_write_transaction_isolation,
 			setup,
 			teardown),
-- 
2.11.0


From 94a5d21849ecce88de143b1fb39895df1faefa67 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 21 Mar 2018 11:38:22 +1300
Subject: [PATCH 53/54] ldb_mdb: handle EBADE from mdb_env_open

Under some circumstances mdb_env_open returns EBADE, we treat this as
indicating the file is not a valid lmdb format file.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 3c2dc980349..f607afee648 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -42,6 +42,7 @@ int ldb_mdb_err_map(int lmdb_err)
 		return LDB_SUCCESS;
 	case EIO:
 		return LDB_ERR_OPERATIONS_ERROR;
+	case EBADE:
 	case MDB_INCOMPATIBLE:
 	case MDB_CORRUPTED:
 	case MDB_INVALID:
@@ -792,7 +793,7 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
 			"Could not create MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
-		return LDB_ERR_OPERATIONS_ERROR;
+		return ldb_mdb_err_map(ret);
 	}
 
 	/* Close when lmdb is released */
-- 
2.11.0


From 3ddec60fd0db9d8500bc28630b8b85bfffe3abc2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 20 Mar 2018 12:58:02 +1300
Subject: [PATCH 54/54] ldb: Release ldb 1.4.0

* New LMDB backend
* Comprehensive tests for index behaviour
* Enforce transactions for writes
* Enforce read lock use for all reads

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ABI/ldb-1.4.0.sigs            | 279 ++++++++++++++++++++++++++++++++++
 lib/ldb/ABI/pyldb-util-1.4.0.sigs     |   2 +
 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs |   2 +
 lib/ldb/wscript                       |   2 +-
 4 files changed, 284 insertions(+), 1 deletion(-)
 create mode 100644 lib/ldb/ABI/ldb-1.4.0.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util-1.4.0.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs

diff --git a/lib/ldb/ABI/ldb-1.4.0.sigs b/lib/ldb/ABI/ldb-1.4.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.4.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.4.0.sigs b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 3da73c16d7c..240179f5563 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 APPNAME = 'ldb'
-VERSION = '1.3.2'
+VERSION = '1.4.0'
 
 blddir = 'bin'
 
-- 
2.11.0

-------------- next part --------------
From 00a0461f622fbb8fd2cbe7900dbe8e5a6e9d5ad3 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 01/25] ldb_mdb: Implement the lmdb backend for ldb

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 711 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.h |  57 ++++
 lib/ldb/ldb_tdb/ldb_tdb.h |   1 +
 lib/ldb/tools/ldbdump.c   | 126 +++++++-
 lib/ldb/wscript           |  89 +++++-
 5 files changed, 977 insertions(+), 7 deletions(-)
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 00000000000..8bc82715390
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,711 @@
+/*
+   ldb database library using mdb back end
+
+   Copyright (C) Jakub Hrozek 2014
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "lmdb.h"
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX		"mdb://"
+#define MDB_URL_PREFIX_SIZE	(sizeof(MDB_URL_PREFIX)-1)
+
+#define MEGABYTE (1024*1024)
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+	switch (lmdb_err) {
+	case MDB_SUCCESS:
+		return LDB_SUCCESS;
+	case EIO:
+		return LDB_ERR_OPERATIONS_ERROR;
+	case MDB_INCOMPATIBLE:
+	case MDB_CORRUPTED:
+	case MDB_INVALID:
+		return LDB_ERR_UNAVAILABLE;
+	case MDB_BAD_TXN:
+	case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+	case MDB_BAD_DBI:
+#endif
+	case MDB_PANIC:
+	case EINVAL:
+		return LDB_ERR_PROTOCOL_ERROR;
+	case MDB_MAP_FULL:
+	case MDB_DBS_FULL:
+	case MDB_READERS_FULL:
+	case MDB_TLS_FULL:
+	case MDB_TXN_FULL:
+	case EAGAIN:
+		return LDB_ERR_BUSY;
+	case MDB_KEYEXIST:
+		return LDB_ERR_ENTRY_ALREADY_EXISTS;
+	case MDB_NOTFOUND:
+	case ENOENT:
+		return LDB_ERR_NO_SUCH_OBJECT;
+	case EACCES:
+		return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+	default:
+		break;
+	}
+	return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+			 int ecode,
+			 const char *file,
+			 int line)
+{
+	int ldb_err = ldb_mdb_err_map(ecode);
+	char *reason = mdb_strerror(ecode);
+	ldb_asprintf_errstring(ldb,
+			       "(%d) - %s at %s:%d",
+			       ecode,
+			       reason,
+			       file,
+			       line);
+	return ldb_err;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+	if (ltx == NULL) {
+		return NULL;
+	}
+
+	return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+	if (lmdb->txlist) {
+		talloc_steal(lmdb->txlist, ltx);
+	}
+
+	DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+	DLIST_REMOVE(lmdb->txlist, ltx);
+	talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+	struct lmdb_trans *ltx;
+
+	ltx = lmdb->txlist;
+	return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+	MDB_txn *txn;
+	if (lmdb->read_txn != NULL) {
+		return lmdb->read_txn;
+	}
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		int ret;
+		ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+		if (ret != 0) {
+			lmdb->error = ret;
+			ldb_asprintf_errstring(lmdb->ldb,
+					       "%s failed: %s\n", __FUNCTION__,
+					       mdb_strerror(ret));
+		}
+		lmdb->read_txn = txn;
+	}
+	return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+		      struct ldb_val key,
+		      struct ldb_val data, int flags)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	int mdb_flags;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+	mdb_data.mv_size = data.length;
+	mdb_data.mv_data = data.data;
+
+	if (flags == TDB_INSERT) {
+		mdb_flags = MDB_NOOVERWRITE;
+	} else if ((flags == TDB_MODIFY)) {
+		/*
+		 * Modifying a record, ensure that it exists.
+		 * This mimics the TDB semantics
+		 */
+		MDB_val value;
+		lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+		if (lmdb->error != MDB_SUCCESS) {
+			if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+				mdb_txn_commit(lmdb->read_txn);
+				lmdb->read_txn = NULL;
+			}
+			return ldb_mdb_error(lmdb->ldb, lmdb->error);
+		}
+		mdb_flags = 0;
+	} else {
+		mdb_flags = 0;
+	}
+
+        lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+
+	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+        lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+		            ldb_kv_traverse_fn fn,
+			    void *ctx)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi = 0;
+	MDB_cursor *cursor = NULL;
+	int ret;
+
+	txn = get_current_txn(lmdb);
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+	if (lmdb->error != MDB_SUCCESS) {
+		goto done;
+	}
+
+	while ((lmdb->error = mdb_cursor_get(
+			cursor, &mdb_key,
+			&mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+		struct ldb_val key = {
+			.length = mdb_key.mv_size,
+			.data = mdb_key.mv_data,
+		};
+		struct ldb_val data = {
+			.length = mdb_data.mv_size,
+			.data = mdb_data.mv_data,
+		};
+
+		ret = fn(ltdb, key, data, ctx);
+		if (ret != 0) {
+			goto done;
+		}
+	}
+	if (lmdb->error == MDB_NOTFOUND) {
+		lmdb->error = MDB_SUCCESS;
+	}
+done:
+	if (cursor != NULL) {
+		mdb_cursor_close(cursor);
+	}
+
+	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+	}
+
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+				  struct ldb_val key,
+				  struct ldb_val key2,
+				  struct ldb_val data,
+				  void *state)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	struct ldb_val copy;
+	int ret = LDB_SUCCESS;
+
+	/*
+	 * Need to take a copy of the data as the delete operation alters the
+	 * data, as it is in private lmdb memory.
+	 */
+	copy.length = data.length;
+	copy.data = talloc_memdup(ltdb, data.data, data.length);
+	if (copy.data == NULL) {
+		lmdb->error = MDB_PANIC;
+		return ldb_oom(lmdb->ldb);
+	}
+
+	lmdb->error = lmdb_delete(ltdb, key);
+	if (lmdb->error != MDB_SUCCESS) {
+		ldb_debug(
+			lmdb->ldb,
+			LDB_DEBUG_ERROR,
+			"Failed to delete %*.*s "
+			"for rekey as %*.*s: %s",
+			(int)key.length, (int)key.length,
+			(const char *)key.data,
+			(int)key2.length, (int)key2.length,
+			(const char *)key.data,
+			mdb_strerror(lmdb->error));
+		ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+		goto done;
+	}
+	lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+	if (lmdb->error != MDB_SUCCESS) {
+		ldb_debug(
+			lmdb->ldb,
+			LDB_DEBUG_ERROR,
+			"Failed to rekey %*.*s as %*.*s: %s",
+			(int)key.length, (int)key.length,
+			(const char *)key.data,
+			(int)key2.length, (int)key2.length,
+			(const char *)key.data,
+			mdb_strerror(lmdb->error));
+		ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+		goto done;
+	}
+
+done:
+	if (copy.data != NULL) {
+		TALLOC_FREE(copy.data);
+		copy.length = 0;
+	}
+
+	/*
+	 * Explicity invalidate the data, as the delete has done this
+	 */
+	data.length = 0;
+	data.data = NULL;
+
+	return ret;
+}
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+			     int (*parser)(struct ldb_val key, struct ldb_val data,
+					   void *private_data),
+			     void *ctx)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	MDB_val mdb_key;
+	MDB_val mdb_data;
+	MDB_txn *txn = NULL;
+	MDB_dbi dbi;
+	struct ldb_val data;
+
+	txn = get_current_txn(lmdb);
+	if (txn == NULL) {
+		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction active");
+		lmdb->error = MDB_PANIC;
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	mdb_key.mv_size = key.length;
+	mdb_key.mv_data = key.data;
+
+        lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+	if (lmdb->error != MDB_SUCCESS) {
+		/* TODO closing a handle should not even be necessary */
+		mdb_dbi_close(lmdb->env, dbi);
+		if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+			mdb_txn_commit(lmdb->read_txn);
+			lmdb->read_txn = NULL;
+		}
+		if (lmdb->error == MDB_NOTFOUND) {
+			return LDB_ERR_NO_SUCH_OBJECT;
+		}
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+	data.data = mdb_data.mv_data;
+	data.length = mdb_data.mv_size;
+
+	/* TODO closing a handle should not even be necessary */
+	mdb_dbi_close(lmdb->env, dbi);
+
+	/* We created a read transaction, commit it */
+	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+	}
+	return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+	void *data = ldb_module_get_private(module);
+	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	lmdb->error = MDB_SUCCESS;
+	if (ltdb->in_transaction == 0 &&
+	    ltdb->read_lock_count == 0) {
+		lmdb->error = mdb_txn_begin(lmdb->env,
+					    NULL,
+					    MDB_RDONLY,
+					    &lmdb->read_txn);
+	}
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	ltdb->read_lock_count++;
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+	void *data = ldb_module_get_private(module);
+	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+		struct lmdb_private *lmdb = ltdb->lmdb_private;
+		mdb_txn_commit(lmdb->read_txn);
+		lmdb->read_txn = NULL;
+		ltdb->read_lock_count--;
+		return LDB_SUCCESS;
+	}
+	ltdb->read_lock_count--;
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	struct lmdb_trans *ltx;
+	struct lmdb_trans *ltx_head;
+	MDB_txn *tx_parent;
+
+	ltx = talloc_zero(lmdb, struct lmdb_trans);
+	if (ltx == NULL) {
+		return ldb_oom(lmdb->ldb);
+	}
+
+	ltx_head = lmdb_private_trans_head(lmdb);
+
+	tx_parent = lmdb_trans_get_tx(ltx_head);
+
+	lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, &ltx->tx);
+	if (lmdb->error != MDB_SUCCESS) {
+		return ldb_mdb_error(lmdb->ldb, lmdb->error);
+	}
+
+	trans_push(lmdb, ltx);
+
+	return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+	struct lmdb_trans *ltx;
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	ltx = lmdb_private_trans_head(lmdb);
+	if (ltx == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	mdb_txn_abort(ltx->tx);
+	trans_finished(lmdb, ltx);
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+	/* No need to prepare a commit */
+	return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+	struct lmdb_trans *ltx;
+	struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+	ltx = lmdb_private_trans_head(lmdb);
+	if (ltx == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	lmdb->error = mdb_txn_commit(ltx->tx);
+	trans_finished(lmdb, ltx);
+
+	return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+	return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+	return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+	return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+	/*
+	 * lmdb does no provide a quick way to determine if the database
+	 * has changed.  This function always returns true.
+	 *
+	 * Note that tdb uses a sequence number that allows this function
+	 * to be implemented efficiently.
+	 */
+	return true;
+}
+
+
+static struct kv_db_ops lmdb_key_value_ops = {
+	.store             = lmdb_store,
+	.delete            = lmdb_delete,
+	.iterate           = lmdb_traverse_fn,
+	.update_in_iterate = lmdb_update_in_iterate,
+	.fetch_and_parse   = lmdb_parse_record,
+	.lock_read         = lmdb_lock_read,
+	.unlock_read       = lmdb_unlock_read,
+	.begin_write       = lmdb_transaction_start,
+	.prepare_write     = lmdb_transaction_prepare_commit,
+	.finish_write      = lmdb_transaction_commit,
+	.abort_write       = lmdb_transaction_cancel,
+	.error             = lmdb_error,
+	.errorstr          = lmdb_errorstr,
+	.name              = lmdb_name,
+	.has_changed       = lmdb_changed,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+	const char *path;
+
+	/* parse the url */
+	if (strchr(url, ':')) {
+		if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+			return NULL;
+		}
+		path = url + MDB_URL_PREFIX_SIZE;
+	} else {
+		path = url;
+	}
+
+	return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+	struct lmdb_trans *ltx = NULL;
+
+	/*
+	 * Close the read transaction if it's open
+	 */
+	if (lmdb->read_txn != NULL) {
+		mdb_txn_abort(lmdb->read_txn);
+	}
+
+	if (lmdb->env == NULL) {
+		return 0;
+	}
+
+	/*
+	 * Abort any currently active transactions
+	 */
+	ltx = lmdb_private_trans_head(lmdb);
+	while (ltx != NULL) {
+		mdb_txn_abort(ltx->tx);
+		trans_finished(lmdb, ltx);
+		ltx = lmdb_private_trans_head(lmdb);
+	}
+
+	mdb_env_close(lmdb->env);
+	lmdb->env = NULL;
+
+	return 0;
+}
+
+static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
+					    struct ldb_context *ldb,
+					    const char *path)
+{
+	struct lmdb_private *lmdb;
+	int ret;
+
+	lmdb = talloc_zero(ldb, struct lmdb_private);
+	if (lmdb == NULL) {
+		return NULL;
+	}
+	lmdb->ldb = ldb;
+
+	ret = mdb_env_create(&lmdb->env);
+	if (ret != 0) {
+		ldb_asprintf_errstring(ldb,
+				"Could not create MDB environment %s: %s\n",
+				path, mdb_strerror(ret));
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	/* Close when lmdb is released */
+	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+	if (ret != 0) {
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	mdb_env_set_maxreaders(lmdb->env, 100000);
+	/* MDB_NOSUBDIR implies there is a separate file called path and a
+	 * separate lockfile called path-lock
+	 */
+	ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+	if (ret != 0) {
+		ldb_asprintf_errstring(ldb,
+				"Could not open DB %s: %s\n",
+				path, mdb_strerror(ret));
+		talloc_free(lmdb);
+		return NULL;
+	}
+
+	return lmdb;
+
+}
+
+static int lmdb_connect(struct ldb_context *ldb, const char *url,
+			unsigned int flags, const char *options[],
+			struct ldb_module **_module)
+{
+	const char *path = NULL;
+	struct lmdb_private *lmdb = NULL;
+	struct ltdb_private *ltdb = NULL;
+
+        /*
+         * We hold locks, so we must use a private event context
+         * on each returned handle
+         */
+        ldb_set_require_private_event_context(ldb);
+
+	path = lmdb_get_path(url);
+	if (path == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+        ltdb = talloc_zero(ldb, struct ltdb_private);
+        if (!ltdb) {
+                ldb_oom(ldb);
+                return LDB_ERR_OPERATIONS_ERROR;
+        }
+	ltdb->kv_ops = &lmdb_key_value_ops;
+
+	lmdb = lmdb_pvt_create(ldb, ldb, path);
+	if (lmdb == NULL) {
+		ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ltdb->lmdb_private = lmdb;
+	if (flags & LDB_FLG_RDONLY) {
+		ltdb->read_only = true;
+	}
+        return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
+
+
+
+int ldb_mdb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 00000000000..18d28e822d7
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,57 @@
+/*
+   ldb database library using mdb back end - transaction operations
+
+   Copyright (C) Jakub Hrozek 2015
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include <lmdb.h>
+
+#include "ldb_private.h"
+
+
+struct lmdb_private {
+	struct ldb_context *ldb;
+	MDB_env *env;
+
+	struct lmdb_trans *txlist;
+
+	struct ldb_mdb_metadata {
+		struct ldb_message *attributes;
+		unsigned seqnum;
+	} *meta;
+	int error;
+	MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+	struct lmdb_trans *next;
+	struct lmdb_trans *prev;
+
+	MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f14666ba88a..7d92804a8e9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
 struct ltdb_private {
 	const struct kv_db_ops *kv_ops;
 	TDB_CONTEXT *tdb;
+	struct lmdb_private *lmdb_private;
 	unsigned int connect_flags;
 	
 	unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59eca4..2da2ca8ec70 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
 #include <ldb.h>
 #include <ldb_private.h>
 
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
 static struct ldb_context *ldb;
 bool show_index = false;
 bool validate_contents = false;
@@ -166,6 +171,116 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
 	return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
 }
 
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+	int ret;
+	struct MDB_env *env = NULL;
+	struct MDB_txn *txn = NULL;
+	MDB_dbi dbi;
+	struct MDB_cursor *cursor = NULL;
+	struct MDB_val key;
+	struct MDB_val data;
+
+	ret = mdb_env_create(&env);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not create MDB environment: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_env_open(env,
+			   fname,
+			   MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY,
+			   0600);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open environment for %s: (%d)  %s\n",
+			fname,
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not start transaction: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_env;
+	}
+
+	ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open database: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_txn;
+	}
+
+	ret = mdb_cursor_open(txn, dbi, &cursor);
+	if (ret != 0) {
+		fprintf(stderr,
+			"Could not open cursor: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_txn;
+	}
+
+	ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+	if (ret != 0 && ret != MDB_NOTFOUND) {
+		fprintf(stderr,
+			"Could not find first record: (%d)  %s\n",
+			ret,
+			mdb_strerror(ret));
+		goto close_cursor;
+	}
+	while (ret != MDB_NOTFOUND) {
+		struct TDB_DATA tkey = {
+			.dptr = key.mv_data,
+			.dsize = key.mv_size
+		};
+		struct TDB_DATA tdata = {
+			.dptr = data.mv_data,
+			.dsize = data.mv_size
+		};
+		traverse_fn(NULL, tkey, tdata, dn);
+		ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+		if (ret != 0 && ret != MDB_NOTFOUND) {
+			fprintf(stderr,
+				"Could not read next record: (%d)  %s\n",
+				ret,
+				mdb_strerror(ret));
+			goto close_cursor;
+		}
+	}
+	ret = 0;
+
+close_cursor:
+	mdb_cursor_close(cursor);
+close_txn:
+	mdb_txn_commit(txn);
+close_env:
+	mdb_env_close(env);
+
+	if (ret != 0) {
+		return 1;
+	}
+	return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+	/* not built with lmdb support */
+	return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
 static void usage( void)
 {
 	printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +344,14 @@ static void usage( void)
 
 	fname = argv[optind];
 
-	return dump_tdb(fname, dn, emergency);
+	rc = dump_lmdb(fname, dn, emergency);
+	if (rc != 0) {
+		rc = dump_tdb(fname, dn, emergency);
+		if (rc != 0) {
+			fprintf(stderr, "Failed to open %s\n", fname);
+			return 1;
+		}
+	}
+	return 0;
+
 }
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 81fcb55e79b..fc216b0831f 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
     srcdir = srcdir + '/..'
 sys.path.insert(0, srcdir + '/buildtools/wafsamba')
 
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
 
 samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
                         lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -92,6 +92,16 @@ def configure(conf):
                                          implied_deps='replace talloc tdb tevent'):
                 conf.define('USING_SYSTEM_LDB', 1)
 
+    if conf.env.standalone_ldb:
+        # Require lmdb support for standalone mode.
+        conf.env.REQUIRE_LMDB = True
+    elif not Options.options.without_ad_dc:
+        # Require lmdb support for addc mode
+        conf.env.REQUIRE_LMDB = True
+    else:
+        conf.env.REQUIRE_LMDB = False
+
+
     if conf.CONFIG_SET('USING_SYSTEM_LDB'):
         v = VERSION.split('.')
         conf.DEFINE('EXPECTED_SYSTEM_LDB_VERSION_MAJOR', int(v[0]))
@@ -110,6 +120,36 @@ def configure(conf):
         if not sys.platform.startswith("openbsd"):
             conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
 
+    # if lmdb support is enabled then we require lmdb
+    # is present, build the mdb back end and enable lmdb support in
+    # the tools.
+    if conf.env.REQUIRE_LMDB:
+        if not conf.CHECK_CFG(package='lmdb',
+                              args='"lmdb >= 0.9.16" --cflags --libs',
+                              msg='Checking for lmdb >= 0.9.16',
+                              mandatory=False):
+            if not conf.CHECK_CODE('''
+                    #if MDB_VERSION_MAJOR == 0 \
+                      && MDB_VERSION_MINOR <= 9 \
+                      && MDB_VERSION_PATCH < 16
+                    #error LMDB too old
+                    #endif
+                    ''',
+                    'HAVE_GOOD_LMDB_VERSION',
+                    headers='lmdb.h',
+                    msg='Checking for lmdb >= 0.9.16 via header check'):
+
+                if conf.env.standalone_ldb:
+                    raise Utils.WafError('ldb requires '
+                                         'lmdb 0.9.16 or later')
+                elif not Options.options.without_ad_dc:
+                    raise Utils.WafError('Samba AD DC requires '
+                                         'lmdb 0.9.16 or later')
+
+        if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+            conf.DEFINE('HAVE_LMDB', '1')
+            conf.env.ENABLE_MDB_BACKEND = True
+
     conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
 
     conf.SAMBA_CONFIG_H()
@@ -321,6 +361,20 @@ def build(bld):
                           private_library=True,
                           deps='tdb ldb')
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            bld.SAMBA_MODULE('ldb_mdb',
+                             bld.SUBDIR('ldb_mdb',
+                                         '''ldb_mdb.c '''),
+                             init_function='ldb_mdb_init',
+                             module_init_name='ldb_init_module',
+                             internal_module=False,
+                             deps='ldb ldb_key_value lmdb',
+                             subsystem='ldb')
+            lmdb_deps = ' ldb_mdb_int'
+        else:
+            lmdb_deps = ''
+
+
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
@@ -338,8 +392,14 @@ def build(bld):
         bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
                          install=False)
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            lmdb_deps = ' lmdb'
+        else:
+            lmdb_deps = ''
         # ldbdump doesn't get installed
-        bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+        bld.SAMBA_BINARY('ldbdump',
+                         'tools/ldbdump.c',
+                         deps='ldb-cmdline ldb' + lmdb_deps,
                          install=False)
 
         bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -365,6 +425,18 @@ def build(bld):
                          deps='cmocka ldb',
                          install=False)
 
+        if bld.CONFIG_SET('HAVE_LMDB'):
+            bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+                             source='tests/ldb_mod_op_test.c',
+                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
+                             deps='cmocka ldb',
+                             install=False)
+            bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
+                             source='tests/ldb_kv_ops_test.c',
+                             cflags='-DTEST_BE=\"mdb\"',
+                             deps='cmocka ldb',
+                             install=False)
+
         bld.SAMBA_BINARY('ldb_msg_test',
                          source='tests/ldb_msg.c',
                          deps='cmocka ldb',
@@ -397,10 +469,15 @@ def test(ctx):
     print("Python testsuite returned %d" % pyret)
 
     cmocka_ret = 0
-    for test_exe in ['ldb_tdb_mod_op_test',
-                     'ldb_tdb_guid_mod_op_test',
-                     'ldb_msg_test',
-                     'ldb_tdb_kv_ops_test']:
+    test_exes = ['ldb_tdb_mod_op_test',
+                 'ldb_tdb_guid_mod_op_test',
+                 'ldb_msg_test',
+                 'ldb_tdb_kv_ops_test']
+
+    if env.ENABLE_MDB_BACKEND:
+        test_exes.append('ldb_mdb_mod_op_test')
+        test_exes.append('ldb_mdb_kv_ops_test')
+    for test_exe in test_exes:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
             cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
 
-- 
2.11.0


From bb767ff21c87b442048cc9c02e4b9f23aca1b7db Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 02/25] ldb: Handle multiple ldb url schemes

Here we use ldb_relative_path more often and allow for the stripping of
ldb:// and mdb://.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb-samba/ldb_wrap.c                            |  6 ++++++
 source4/dsdb/samdb/ldb_modules/partition_metadata.c | 20 +++++++++-----------
 source4/dsdb/samdb/ldb_modules/schema_load.c        | 19 +++++--------------
 3 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 143e128d5d7..4dc5ca6da2a 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -356,6 +356,12 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
 	if (strncmp("tdb://", base_url, 6) == 0) {
 		base_url = base_url+6;
 	}
+	else if (strncmp("mdb://", base_url, 6) == 0) {
+		base_url = base_url+6;
+	}
+	else if (strncmp("ldb://", base_url, 6) == 0) {
+		base_url = base_url+6;
+	}
 	path = talloc_strdup(mem_ctx, base_url);
 	if (path == NULL) {
 		return NULL;
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index e3ad0d8c6c2..510228e319b 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -18,6 +18,7 @@
 */
 
 #include "dsdb/samdb/ldb_modules/partition.h"
+#include "lib/ldb-samba/ldb_wrap.h"
 #include "system/filesys.h"
 
 #define LDB_METADATA_SEQ_NUM	"SEQ_NUM"
@@ -185,7 +186,6 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 	TALLOC_CTX *tmp_ctx;
 	struct partition_private_data *data;
 	struct loadparm_context *lp_ctx;
-	const char *sam_name;
 	char *filename, *dirname;
 	int open_flags;
 	struct stat statbuf;
@@ -202,15 +202,11 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 		return ldb_module_oom(module);
 	}
 
-	sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
-	if (!sam_name) {
-		talloc_free(tmp_ctx);
-		return ldb_operr(ldb);
-	}
-	if (strncmp("tdb://", sam_name, 6) == 0) {
-		sam_name += 6;
-	}
-	filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
+	/* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+	filename = ldb_relative_path(ldb,
+				     tmp_ctx,
+				     "sam.ldb.d/metadata.tdb");
+
 	if (!filename) {
 		talloc_free(tmp_ctx);
 		return ldb_oom(ldb);
@@ -222,7 +218,9 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
 
 		/* While provisioning, sam.ldb.d directory may not exist,
 		 * so create it. Ignore errors, if it already exists. */
-		dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name);
+		dirname = ldb_relative_path(ldb,
+					    tmp_ctx,
+					    "sam.ldb.d");
 		if (!dirname) {
 			talloc_free(tmp_ctx);
 			return ldb_oom(ldb);
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 2099fac1159..e21a5a5a7e2 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -32,6 +32,7 @@
 #include <tdb.h>
 #include "lib/tdb_wrap/tdb_wrap.h"
 #include "dsdb/samdb/ldb_modules/util.h"
+#include "lib/ldb-samba/ldb_wrap.h"
 
 #include "system/filesys.h"
 struct schema_load_private_data {
@@ -58,7 +59,6 @@ static int schema_metadata_open(struct ldb_module *module)
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 	TALLOC_CTX *tmp_ctx;
 	struct loadparm_context *lp_ctx;
-	const char *sam_name;
 	char *filename;
 	int open_flags;
 	struct stat statbuf;
@@ -74,19 +74,10 @@ static int schema_metadata_open(struct ldb_module *module)
 		return ldb_module_oom(module);
 	}
 
-	sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
-	if (!sam_name) {
-		talloc_free(tmp_ctx);
-		return ldb_operr(ldb);
-	}
-	if (strncmp("tdb://", sam_name, 6) == 0) {
-		sam_name += 6;
-	}
-	filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
-	if (!filename) {
-		talloc_free(tmp_ctx);
-		return ldb_oom(ldb);
-	}
+	/* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+	filename = ldb_relative_path(ldb,
+				     tmp_ctx,
+				     "sam.ldb.d/metadata.tdb");
 
 	open_flags = O_RDWR;
 	if (stat(filename, &statbuf) != 0) {
-- 
2.11.0


From 9916a587dfeff3535116c39fed2bdf925e4bb988 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 03/25] ldb: Introduce new ldb:// prefix to allow failover

If you try to open with tdb first you find that you can clobber a
database due to the default tdb behaviour.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_ldb/ldb_ldb.c      | 78 +++++++++++++++++++++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.c      | 79 +++++++++++++++++++++++++-----------------
 lib/ldb/ldb_mdb/ldb_mdb.h      |  3 ++
 lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 +++++++++++++++++
 lib/ldb/wscript                | 19 ++++++++--
 5 files changed, 176 insertions(+), 34 deletions(-)
 create mode 100644 lib/ldb/ldb_ldb/ldb_ldb.c
 create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c

diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
new file mode 100644
index 00000000000..a63967ff8d6
--- /dev/null
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -0,0 +1,78 @@
+/*
+ * ldb connection and module initialisation
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#include "ldb_private.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+
+/*
+  connect to the database
+*/
+static int lldb_connect(struct ldb_context *ldb,
+			const char *url,
+			unsigned int flags,
+			const char *options[],
+			struct ldb_module **module)
+{
+	const char *path;
+	int ret;
+
+	/*
+	 * Check and remove the url prefix
+	 */
+	if (strchr(url, ':')) {
+		if (strncmp(url, "ldb://", 6) != 0) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR,
+				  "Invalid ldb URL '%s'", url);
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		path = url+6;
+	} else {
+		path = url;
+	}
+
+	/*
+	 * Don't create the database if it's not there
+	 */
+	flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+	/*
+	 * Try opening the database as an lmdb
+	 */
+	ret = lmdb_connect(ldb, path, flags, options, module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (ret == LDB_ERR_UNAVAILABLE) {
+		/*
+		 * Not mdb so try as tdb
+		 */
+		ret = ltdb_connect(ldb, path, flags, options, module);
+	}
+#else
+	ret = ltdb_connect(ldb, path, flags, options, module);
+#endif
+	return ret;
+}
+
+int ldb_ldb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("ldb", lldb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 8bc82715390..314c78ff626 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -614,26 +614,30 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 	return 0;
 }
 
-static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
-					    struct ldb_context *ldb,
-					    const char *path)
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+				  struct ldb_context *ldb,
+				  const char *path,
+				  unsigned int flags,
+				  struct lmdb_private *lmdb)
 {
-	struct lmdb_private *lmdb;
 	int ret;
+	unsigned int mdb_flags;
 
-	lmdb = talloc_zero(ldb, struct lmdb_private);
-	if (lmdb == NULL) {
-		return NULL;
+	if (flags & LDB_FLG_DONT_CREATE_DB) {
+		struct stat st;
+		if (stat(path, &st) != 0) {
+			return LDB_ERR_UNAVAILABLE;
+		}
 	}
-	lmdb->ldb = ldb;
 
 	ret = mdb_env_create(&lmdb->env);
 	if (ret != 0) {
-		ldb_asprintf_errstring(ldb,
-				"Could not create MDB environment %s: %s\n",
-				path, mdb_strerror(ret));
-		talloc_free(lmdb);
-		return NULL;
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not create MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
 	/* Close when lmdb is released */
@@ -641,34 +645,45 @@ static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
 
 	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
 	if (ret != 0) {
-		talloc_free(lmdb);
-		return NULL;
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not open MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return ldb_mdb_err_map(ret);
 	}
 
 	mdb_env_set_maxreaders(lmdb->env, 100000);
 	/* MDB_NOSUBDIR implies there is a separate file called path and a
 	 * separate lockfile called path-lock
 	 */
-	ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+	mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+	if (flags & LDB_FLG_RDONLY) {
+		mdb_flags |= MDB_RDONLY;
+	}
+	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
 				"Could not open DB %s: %s\n",
 				path, mdb_strerror(ret));
 		talloc_free(lmdb);
-		return NULL;
+		return ldb_mdb_err_map(ret);
 	}
 
-	return lmdb;
+	return LDB_SUCCESS;
 
 }
 
-static int lmdb_connect(struct ldb_context *ldb, const char *url,
-			unsigned int flags, const char *options[],
-			struct ldb_module **_module)
+int lmdb_connect(struct ldb_context *ldb,
+		 const char *url,
+		 unsigned int flags,
+		 const char *options[],
+		 struct ldb_module **_module)
 {
 	const char *path = NULL;
 	struct lmdb_private *lmdb = NULL;
 	struct ltdb_private *ltdb = NULL;
+	int ret;
 
         /*
          * We hold locks, so we must use a private event context
@@ -687,12 +702,19 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
                 ldb_oom(ldb);
                 return LDB_ERR_OPERATIONS_ERROR;
         }
-	ltdb->kv_ops = &lmdb_key_value_ops;
 
-	lmdb = lmdb_pvt_create(ldb, ldb, path);
+	lmdb = talloc_zero(ldb, struct lmdb_private);
 	if (lmdb == NULL) {
-		ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
-		return LDB_ERR_OPERATIONS_ERROR;
+		TALLOC_FREE(ltdb);
+                ldb_oom(ldb);
+                return LDB_ERR_OPERATIONS_ERROR;
+	}
+	lmdb->ldb = ldb;
+	ltdb->kv_ops = &lmdb_key_value_ops;
+
+	ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+	if (ret != LDB_SUCCESS) {
+		return ret;
 	}
 
 	ltdb->lmdb_private = lmdb;
@@ -702,10 +724,3 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
         return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
 }
 
-
-
-int ldb_mdb_init(const char *version)
-{
-	LDB_MODULE_CHECK_VERSION(version);
-	return ldb_register_backend("mdb", lmdb_connect, false);
-}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 18d28e822d7..e62e353b54e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -53,5 +53,8 @@ struct lmdb_trans {
 };
 
 int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+		 unsigned int flags, const char *options[],
+		 struct ldb_module **_module);
 
 #endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 00000000000..339c3f22b2a
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+   ldb database library using mdb back end
+
+   Copyright (C) Jakub Hrozek 2014
+   Copyright (C) Catalyst.Net Ltd 2017
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+	LDB_MODULE_CHECK_VERSION(version);
+	return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index fc216b0831f..ec5d8eff97f 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -364,17 +364,32 @@ def build(bld):
         if bld.CONFIG_SET('HAVE_LMDB'):
             bld.SAMBA_MODULE('ldb_mdb',
                              bld.SUBDIR('ldb_mdb',
-                                         '''ldb_mdb.c '''),
+                                        '''ldb_mdb_init.c'''),
                              init_function='ldb_mdb_init',
                              module_init_name='ldb_init_module',
                              internal_module=False,
-                             deps='ldb ldb_key_value lmdb',
+                             deps='ldb ldb_key_value ldb_mdb_int',
                              subsystem='ldb')
+
+            bld.SAMBA_LIBRARY('ldb_mdb_int',
+                              bld.SUBDIR('ldb_mdb',
+                                         '''ldb_mdb.c '''),
+                              private_library=True,
+                              deps='ldb lmdb ldb_key_value')
             lmdb_deps = ' ldb_mdb_int'
         else:
             lmdb_deps = ''
 
 
+        bld.SAMBA_MODULE('ldb_ldb',
+                         bld.SUBDIR('ldb_ldb',
+                                    '''ldb_ldb.c'''),
+                         init_function='ldb_ldb_init',
+                         module_init_name='ldb_init_module',
+                         internal_module=False,
+                         deps='ldb ldb_key_value' + lmdb_deps,
+                         subsystem='ldb')
+
         # have a separate subsystem for common/ldb.c, so it can rebuild
         # for install with a different -DLDB_MODULESDIR=
         bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
-- 
2.11.0


From 18937d9c6460a55a4064682134175d001ba71d5c Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 5 Jan 2018 09:28:54 +1300
Subject: [PATCH 04/25] tests: Replace some references to tdb with ldb://

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 python/samba/tests/encrypted_secrets.py      |  2 +-
 source4/torture/drs/python/samba_tool_drs.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/python/samba/tests/encrypted_secrets.py b/python/samba/tests/encrypted_secrets.py
index 3b6934f0ccb..114f3b91108 100644
--- a/python/samba/tests/encrypted_secrets.py
+++ b/python/samba/tests/encrypted_secrets.py
@@ -54,7 +54,7 @@ class EncryptedSecretsTests(TestCase):
         backend_subpath = os.path.join("sam.ldb.d",
                                        backend_filename)
         backend_path = self.lp.private_path(backend_subpath)
-        backenddb = ldb.Ldb(backend_path)
+        backenddb = ldb.Ldb("ldb://" + backend_path, flags=ldb.FLG_DONT_CREATE_DB)
 
         dn = "CN=Administrator,CN=Users,%s" % basedn
 
diff --git a/source4/torture/drs/python/samba_tool_drs.py b/source4/torture/drs/python/samba_tool_drs.py
index fcc681dc9e4..502c8096603 100644
--- a/source4/torture/drs/python/samba_tool_drs.py
+++ b/source4/torture/drs/python/samba_tool_drs.py
@@ -282,7 +282,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
                                    self.dc1,
                                    self.cmdline_creds,
                                    self.tempdir))
-        ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+        ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
         nc_name = ldb_rootdse["defaultNamingContext"]
         ds_name = ldb_rootdse["dsServiceName"]
         ldap_service_name = str(server_rootdse["ldapServiceName"][0])
@@ -291,7 +291,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
         self.assertEqual(ds_name, server_ds_name)
         self.assertEqual(ldap_service_name, server_ldap_service_name)
 
-        samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+        samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
                                           ldap_only=False, lp=self.get_loadparm())
         def get_krbtgt_pw():
             krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
@@ -346,13 +346,13 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
                                    self.dc1,
                                    self.cmdline_creds,
                                    self.tempdir))
-        ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+        ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
         nc_name = ldb_rootdse["defaultNamingContext"]
         config_nc_name = ldb_rootdse["configurationNamingContext"]
         ds_name = ldb_rootdse["dsServiceName"]
         ldap_service_name = str(server_rootdse["ldapServiceName"][0])
 
-        samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+        samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
                                           ldap_only=False, lp=self.get_loadparm())
         krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
         self.assertIsNotNone(krbtgt_pw)
@@ -381,7 +381,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
         self.assertRaises(samba.tests.BlackboxProcessError, demote_self)
 
         # While we have this cloned, try demoting the other server on the clone
-        out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H %s/private/sam.ldb"
+        out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H ldb://%s/private/sam.ldb"
                                 % (self.dc2,
                                    self.tempdir))
 
-- 
2.11.0


From 6ca5da26bdb4a2e061f78aa0bce8f554e9495e3f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 12:40:01 +1300
Subject: [PATCH 05/25] tests/dlz_bind9: support for multiple db types

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 source4/torture/dns/dlz_bind9.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source4/torture/dns/dlz_bind9.c b/source4/torture/dns/dlz_bind9.c
index 2234e7a23f6..42b104e070c 100644
--- a/source4/torture/dns/dlz_bind9.c
+++ b/source4/torture/dns/dlz_bind9.c
@@ -58,7 +58,7 @@ static char *test_dlz_bind9_binddns_dir(struct torture_context *tctx,
 					const char *file)
 {
 	return talloc_asprintf(tctx,
-			       "%s/%s",
+			       "ldb://%s/%s",
 			       lpcfg_binddns_dir(tctx->lp_ctx),
 			       file);
 }
-- 
2.11.0


From b3f047d496df248889e4326f6d45f0af85a34b21 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 06/25] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb

This is used in selftest with 'ldb:nosync = true'.

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 314c78ff626..a34cf35a7d5 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -661,6 +661,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 	if (flags & LDB_FLG_RDONLY) {
 		mdb_flags |= MDB_RDONLY;
 	}
+	if (flags & LDB_FLG_NOSYNC) {
+		mdb_flags |= MDB_NOSYNC;
+	}
 	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
-- 
2.11.0


From 64bdc3e018f70aa3380fecfc7816b57cd4984d8d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 07/25] tests: Add tests to check for max key length and DB
 size

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_lmdb_size_test.c | 213 ++++++++++++++++++++++++++++++++
 lib/ldb/tests/ldb_lmdb_test.c      | 243 +++++++++++++++++++++++++++++++++++++
 lib/ldb/wscript                    |  12 ++
 3 files changed, 468 insertions(+)
 create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
 create mode 100644 lib/ldb/tests/ldb_lmdb_test.c

diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 00000000000..506c586289e
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,213 @@
+/*
+ * lmdb backend specific tests for ldb
+ * Tests for truncated index keys
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE  "mdb"
+
+struct ldbtest_ctx {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+
+	const char *dbfile;
+	const char *lockfile;   /* lockfile is separate */
+
+	const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+	int ret;
+
+	errno = 0;
+	ret = unlink(test_ctx->lockfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+
+	errno = 0;
+	ret = unlink(test_ctx->dbfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+
+	test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+	assert_non_null(test_ctx);
+
+	test_ctx->ev = tevent_context_init(test_ctx);
+	assert_non_null(test_ctx->ev);
+
+	test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+	assert_non_null(test_ctx->ldb);
+
+	test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+	assert_non_null(test_ctx->dbfile);
+
+	test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+					     test_ctx->dbfile);
+	assert_non_null(test_ctx->lockfile);
+
+	test_ctx->dbpath = talloc_asprintf(test_ctx,
+			TEST_BE"://%s", test_ctx->dbfile);
+	assert_non_null(test_ctx->dbpath);
+
+	unlink_old_db(test_ctx);
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+
+	unlink_old_db(test_ctx);
+	talloc_free(test_ctx);
+	return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	ldbtest_noconn_teardown((void **) &test_ctx);
+	return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+	int ret, x;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	const int MB = 1024 * 1024;
+	char *blob = NULL;
+
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+
+	blob = talloc_zero_size(tmp_ctx, (MB + 1));
+	assert_non_null(blob);
+	memset(blob, 'x', MB);
+
+
+	for (x = 0; x < 6144; x++) {
+		msg = ldb_msg_new(tmp_ctx);
+		assert_non_null(msg);
+
+		msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+		assert_non_null(msg->dn);
+
+		ldb_transaction_start(test_ctx->ldb);
+		ret = ldb_msg_add_string(msg, "blob", blob);
+		assert_int_equal(ret, 0);
+
+		ret = ldb_add(test_ctx->ldb, msg);
+		assert_int_equal(ret, 0);
+		ldb_transaction_commit(test_ctx->ldb);
+
+		TALLOC_FREE(msg);
+	}
+	talloc_free(tmp_ctx);
+	{
+		struct stat s;
+		ret = stat(test_ctx->dbfile, &s);
+		assert_int_equal(ret, 0);
+		assert_true(s.st_size > (6144LL * MB));
+	}
+}
+
+int main(int argc, const char **argv)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(
+			test_db_size_gt_4GB,
+			ldbtest_setup,
+			ldbtest_teardown),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 00000000000..2902eaf0533
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,243 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ *  Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied  from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE  "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+	struct tevent_context *ev;
+	struct ldb_context *ldb;
+
+	const char *dbfile;
+	const char *lockfile;   /* lockfile is separate */
+
+	const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+	int ret;
+
+	errno = 0;
+	ret = unlink(test_ctx->lockfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+
+	errno = 0;
+	ret = unlink(test_ctx->dbfile);
+	if (ret == -1 && errno != ENOENT) {
+		fail();
+	}
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+
+	test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+	assert_non_null(test_ctx);
+
+	test_ctx->ev = tevent_context_init(test_ctx);
+	assert_non_null(test_ctx->ev);
+
+	test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+	assert_non_null(test_ctx->ldb);
+
+	test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+	assert_non_null(test_ctx->dbfile);
+
+	test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+					     test_ctx->dbfile);
+	assert_non_null(test_ctx->lockfile);
+
+	test_ctx->dbpath = talloc_asprintf(test_ctx,
+			TEST_BE"://%s", test_ctx->dbfile);
+	assert_non_null(test_ctx->dbpath);
+
+	unlink_old_db(test_ctx);
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+
+	unlink_old_db(test_ctx);
+	talloc_free(test_ctx);
+	return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	ldbtest_noconn_teardown((void **) &test_ctx);
+	return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, 0);
+
+	talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_eq_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_gt_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index ec5d8eff97f..d9f47c798bf 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -446,6 +446,17 @@ def build(bld):
                              cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
                              deps='cmocka ldb',
                              install=False)
+            
+            bld.SAMBA_BINARY('ldb_lmdb_test',
+                             source='tests/ldb_lmdb_test.c',
+                             deps='cmocka ldb',
+                             install=False)
+
+            bld.SAMBA_BINARY('ldb_lmdb_size_test',
+                             source='tests/ldb_lmdb_size_test.c',
+                             deps='cmocka ldb',
+                             install=False)
+
             bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
                              source='tests/ldb_kv_ops_test.c',
                              cflags='-DTEST_BE=\"mdb\"',
@@ -491,6 +502,7 @@ def test(ctx):
 
     if env.ENABLE_MDB_BACKEND:
         test_exes.append('ldb_mdb_mod_op_test')
+        test_exes.append('ldb_lmdb_test')
         test_exes.append('ldb_mdb_kv_ops_test')
     for test_exe in test_exes:
             cmd = os.path.join(Utils.g_module.blddir, test_exe)
-- 
2.11.0


From e7f9f0cfd87d5d6dc2dc74419ebb8a27969dd70e Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 08/25] ldb_mdb: Store pid to change destructor on fork

Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
 lib/ldb/ldb_mdb/ldb_mdb.h |  2 ++
 2 files changed, 24 insertions(+)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index a34cf35a7d5..711700c2697 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -587,6 +587,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 {
 	struct lmdb_trans *ltx = NULL;
 
+	/* Check if this is a forked child */
+	if (getpid() != lmdb->pid) {
+		int fd = 0;
+		/*
+		 * We cannot call mdb_env_close or commit any transactions,
+		 * otherwise they might appear finished in the parent.
+		 *
+		 */
+
+		if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+			close(fd);
+		}
+
+		/* Remove the pointer, so that no access should occur */
+		lmdb->env = NULL;
+
+		return 0;
+	}
+
 	/*
 	 * Close the read transaction if it's open
 	 */
@@ -673,6 +692,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 		return ldb_mdb_err_map(ret);
 	}
 
+	/* Store the original pid during the LMDB open */
+	lmdb->pid = getpid();
+
 	return LDB_SUCCESS;
 
 }
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index e62e353b54e..7e0729cc7af 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -43,6 +43,8 @@ struct lmdb_private {
 	int error;
 	MDB_txn *read_txn;
 
+	pid_t pid;
+
 };
 
 struct lmdb_trans {
-- 
2.11.0


From d531ce6f9c7c38cbca2803dffc741f672df0d86f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 09/25] lib ldb tests: Run api and index test on tdb and lmdb

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py   | 87 ++++++++++++++++++++++++++++++++++++++++++-
 lib/ldb/tests/python/index.py | 11 ++++++
 2 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 12409a8c991..9a2997f5b48 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -633,6 +633,16 @@ class SimpleLdb(LdbBaseTest):
         l = ldb.Ldb(self.url(), flags=self.flags())
         self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
 
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(SimpleLdbLmdb, self).setUp()
+
+    def tearDown(self):
+        super(SimpleLdbLmdb, self).tearDown()
+
 class SearchTests(LdbBaseTest):
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -1057,6 +1067,17 @@ class SearchTests(LdbBaseTest):
         self.assertEqual(len(res11), 1)
 
 
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(SearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(SearchTestsLmdb, self).tearDown()
+
+
 class IndexedSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1152,6 +1173,35 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
         self.IDXGUID = True
         self.IDXONE = True
 
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
 
 class AddModifyTests(LdbBaseTest):
     def tearDown(self):
@@ -1296,6 +1346,15 @@ class AddModifyTests(LdbBaseTest):
                     "objectUUID": b"0123456789abcde3"})
 
 
+class AddModifyTestsLmdb(AddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(AddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(AddModifyTestsLmdb, self).tearDown()
+
 class IndexedAddModifyTests(AddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
@@ -1407,6 +1466,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
         self.l.transaction_commit()
         super(TransIndexedAddModifyTests, self).tearDown()
 
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
 
 class BadIndexTests(LdbBaseTest):
     def setUp(self):
@@ -1567,7 +1643,6 @@ class GUIDBadIndexTests(BadIndexTests):
 
         super(GUIDBadIndexTests, self).setUp()
 
-
 class DnTests(TestCase):
 
     def setUp(self):
@@ -2429,6 +2504,16 @@ class LdbResultTests(LdbBaseTest):
         self.assertEqual(got_pid, pid)
 
 
+class LdbResultTestsLmdb(LdbResultTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(LdbResultTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(LdbResultTestsLmdb, self).tearDown()
+
+
 class BadTypeTests(TestCase):
     def test_control(self):
         l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 9b9e4f3469f..3379fb9374f 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -1280,6 +1280,17 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
             code = e.args[0]
             self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
 
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+    def setUp(self):
+        self.prefix = MDB_PREFIX
+        super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+    def tearDown(self):
+        super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
 if __name__ == '__main__':
     import unittest
     unittest.TestProgram()
-- 
2.11.0


From 285814ffe42c20d0ccea92ba93af79e3f2293b50 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:10 +1300
Subject: [PATCH 10/25] ldb: Allow tests to operate on ldb_mdb after new
 restrictions

The tests were working fine on ldb_mdb but only in the mode without
the GUID index.  This mode is not OK for production so has been banned
but this means we need to rework the tests to set an objectGUID on
each record.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++---------------
 1 file changed, 121 insertions(+), 64 deletions(-)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 9a2997f5b48..bb2c918e9f6 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -77,6 +77,10 @@ class SimpleLdb(LdbBaseTest):
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
         self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+        try:
+            self.ldb.add(self.index)
+        except AttributeError:
+            pass
 
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -165,6 +169,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo1")
         m["b"] = [b"a"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         self.assertRaises(ldb.LdbError, lambda: l.delete(m.dn, ["search_options:1:2"]))
         l.delete(m.dn)
@@ -177,6 +182,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo3")
         m["b"] = ["a"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             self.assertTrue(ldb.Dn(l, "dc=foo3") in l)
@@ -205,6 +211,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -245,6 +252,7 @@ class SimpleLdb(LdbBaseTest):
         m1 = ldb.Message()
         m1.dn = ldb.Dn(l, "dc=foo4")
         m1["bla"] = b"bla"
+        m1["objectUUID"] = b"0123456789abcdef"
         l.add(m1)
         try:
             s = l.search_iterator()
@@ -261,6 +269,7 @@ class SimpleLdb(LdbBaseTest):
             m2 = ldb.Message()
             m2.dn = ldb.Dn(l, "dc=foo5")
             m2["bla"] = b"bla"
+            m2["objectUUID"] = b"0123456789abcdee"
             l.add(m2)
 
             s = l.search_iterator()
@@ -317,6 +326,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo4")
         m["bla"] = "bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -335,7 +345,8 @@ class SimpleLdb(LdbBaseTest):
     def test_add_dict(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
-             "bla": b"bla"}
+             "bla": b"bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -346,7 +357,8 @@ class SimpleLdb(LdbBaseTest):
     def test_add_dict_text(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
         m = {"dn": ldb.Dn(l, "dc=foo5"),
-             "bla": "bla"}
+             "bla": "bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -356,7 +368,8 @@ class SimpleLdb(LdbBaseTest):
 
     def test_add_dict_string_dn(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
-        m = {"dn": "dc=foo6", "bla": b"bla"}
+        m = {"dn": "dc=foo6", "bla": b"bla",
+             "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -366,7 +379,8 @@ class SimpleLdb(LdbBaseTest):
 
     def test_add_dict_bytes_dn(self):
         l = ldb.Ldb(self.url(), flags=self.flags())
-        m = {"dn": b"dc=foo6", "bla": b"bla"}
+        m = {"dn": b"dc=foo6", "bla": b"bla",
+              "objectUUID": b"0123456789abcdef"}
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -379,6 +393,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo7")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         try:
@@ -392,6 +407,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=foo8")
         m["bla"] = b"bla"
+        m["objectUUID"] = b"0123456789abcdef"
         self.assertEqual(len(l.search()), 0)
         l.add(m)
         self.assertEqual(len(l.search()), 1)
@@ -406,14 +422,17 @@ class SimpleLdb(LdbBaseTest):
         self.assertEqual(0, len(l.search()))
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=empty")
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search()
         self.assertEqual(1, len(rm))
-        self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+        self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                         set(rm[0].keys()))
 
         rm = l.search(m.dn)
         self.assertEqual(1, len(rm))
-        self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+        self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                         set(rm[0].keys()))
         rm = l.search(m.dn, attrs=["blah"])
         self.assertEqual(1, len(rm))
         self.assertEqual(0, len(rm[0]))
@@ -423,6 +442,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search(m.dn)[0]
         self.assertEqual([b"1234"], list(rm["bla"]))
@@ -434,7 +454,8 @@ class SimpleLdb(LdbBaseTest):
             l.modify(m)
             rm = l.search(m.dn)
             self.assertEqual(1, len(rm))
-            self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+            self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                             set(rm[0].keys()))
             rm = l.search(m.dn, attrs=["bla"])
             self.assertEqual(1, len(rm))
             self.assertEqual(0, len(rm[0]))
@@ -446,6 +467,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modifydelete")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         rm = l.search(m.dn)[0]
         self.assertEqual(["1234"], list(rm.text["bla"]))
@@ -457,7 +479,8 @@ class SimpleLdb(LdbBaseTest):
             l.modify(m)
             rm = l.search(m.dn)
             self.assertEqual(1, len(rm))
-            self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+            self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+                             set(rm[0].keys()))
             rm = l.search(m.dn, attrs=["bla"])
             self.assertEqual(1, len(rm))
             self.assertEqual(0, len(rm[0]))
@@ -469,6 +492,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -477,7 +501,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"1234", b"456"], list(rm["bla"]))
         finally:
             l.delete(ldb.Dn(l, "dc=add"))
@@ -487,6 +511,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -495,7 +520,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["1234", "456"], list(rm.text["bla"]))
         finally:
             l.delete(ldb.Dn(l, "dc=add"))
@@ -505,6 +530,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m["bla"] = [b"1234", b"456"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -513,7 +539,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"789"], list(rm["bla"]))
             rm = l.search(m.dn, attrs=["bla"])[0]
             self.assertEqual(1, len(rm))
@@ -525,6 +551,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=modify2")
         m.text["bla"] = ["1234", "456"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -533,7 +560,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["789"], list(rm.text["bla"]))
             rm = l.search(m.dn, attrs=["bla"])[0]
             self.assertEqual(1, len(rm))
@@ -545,6 +572,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m["bla"] = [b"1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -553,7 +581,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual([b"1234", b"456"], list(rm["bla"]))
 
             # Now create another modify, but switch the flags before we do it
@@ -571,6 +599,7 @@ class SimpleLdb(LdbBaseTest):
         m = ldb.Message()
         m.dn = ldb.Dn(l, "dc=add")
         m.text["bla"] = ["1234"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         try:
             m = ldb.Message()
@@ -579,7 +608,7 @@ class SimpleLdb(LdbBaseTest):
             self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
             l.modify(m)
             rm = l.search(m.dn)[0]
-            self.assertEqual(2, len(rm))
+            self.assertEqual(3, len(rm))
             self.assertEqual(["1234", "456"], list(rm.text["bla"]))
 
             # Now create another modify, but switch the flags before we do it
@@ -597,6 +626,7 @@ class SimpleLdb(LdbBaseTest):
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo9"))
         m["foo"] = [b"bar"]
+        m["objectUUID"] = b"0123456789abcdef"
         l.add(m)
         l.transaction_commit()
         l.delete(m.dn)
@@ -606,6 +636,7 @@ class SimpleLdb(LdbBaseTest):
         l.transaction_start()
         m = ldb.Message(ldb.Dn(l, "dc=foo10"))
         m["foo"] = [b"bar"]
+        m["objectUUID"] = b"0123456789abcdee"
         l.add(m)
         l.transaction_cancel()
         self.assertEqual(0, len(l.search(ldb.Dn(l, "dc=foo10"))))
@@ -625,6 +656,7 @@ class SimpleLdb(LdbBaseTest):
             "cN" : b"LDAPtestUSER",
             "givenname" : b"ldap",
             "displayname" : b"foo\0bar",
+            "objectUUID" : b"0123456789abcdef"
         })
         res = l.search(expression="(dn=dc=somedn)")
         self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
@@ -638,6 +670,10 @@ class SimpleLdbLmdb(SimpleLdb):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXONE": [b"1"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(SimpleLdbLmdb, self).setUp()
 
     def tearDown(self):
@@ -659,6 +695,10 @@ class SearchTests(LdbBaseTest):
         self.l = ldb.Ldb(self.url(),
                          flags=self.flags(),
                          options=["modules:rdn_name"])
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
 
         self.l.add({"dn": "@ATTRIBUTES",
                     "DC": "CASE_INSENSITIVE"})
@@ -670,7 +710,7 @@ class SearchTests(LdbBaseTest):
 
         self.l.add({"dn": "DC=SAMBA,DC=ORG",
                     "name": b"samba.org",
-                    "objectUUID": b"0123456789abcddf"})
+                    "objectUUID": b"0123456789abcdef"})
         self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
                     "name": b"Admins",
                     "x": "z", "y": "a",
@@ -1072,6 +1112,9 @@ class SearchTestsLmdb(SearchTests):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(SearchTestsLmdb, self).setUp()
 
     def tearDown(self):
@@ -1129,12 +1172,12 @@ class GUIDIndexedSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedSearchTests, self).setUp()
 
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDXGUID = True
         self.IDXONE = True
 
@@ -1143,15 +1186,14 @@ class GUIDIndexedDNFilterSearchTests(SearchTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedDNFilterSearchTests, self).setUp()
         self.l.add({"dn": "@OPTIONS",
                     "disallowDNFilter": "TRUE"})
         self.disallowDNFilter = True
-
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDX = True
         self.IDXGUID = True
 
@@ -1159,16 +1201,14 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
     """Test searches using the index including @IDXONE, to ensure
        the index doesn't break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDAndOneLevelIndexedSearchTests, self).setUp()
         self.l.add({"dn": "@OPTIONS",
                     "disallowDNFilter": "TRUE"})
         self.disallowDNFilter = True
-
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou"],
-                    "@IDXONE": [b"1"],
-                    "@IDXGUID": [b"objectUUID"],
-                    "@IDX_DN_GUID": [b"GUID"]})
         self.IDX = True
         self.IDXGUID = True
         self.IDXONE = True
@@ -1218,6 +1258,11 @@ class AddModifyTests(LdbBaseTest):
         self.l = ldb.Ldb(self.url(),
                          flags=self.flags(),
                          options=["modules:rdn_name"])
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
+
         self.l.add({"dn": "DC=SAMBA,DC=ORG",
                     "name": b"samba.org",
                     "objectUUID": b"0123456789abcdef"})
@@ -1346,23 +1391,15 @@ class AddModifyTests(LdbBaseTest):
                     "objectUUID": b"0123456789abcde3"})
 
 
-class AddModifyTestsLmdb(AddModifyTests):
-
-    def setUp(self):
-        self.prefix = MDB_PREFIX
-        super(AddModifyTestsLmdb, self).setUp()
-
-    def tearDown(self):
-        super(AddModifyTestsLmdb, self).tearDown()
-
 class IndexedAddModifyTests(AddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        if not hasattr(self, 'index'):
+            self.index = {"dn": "@INDEXLIST",
+                          "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
+                          "@IDXONE": [b"1"]}
         super(IndexedAddModifyTests, self).setUp()
-        self.l.add({"dn": "@INDEXLIST",
-                    "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
-                    "@IDXONE": [b"1"]})
 
     def test_duplicate_GUID(self):
         try:
@@ -1436,14 +1473,12 @@ class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
     """Test searches using the index, to ensure the index doesn't
        break things"""
     def setUp(self):
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXATTR": [b"x", b"y", b"ou"],
+                      "@IDXONE": [b"1"],
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(GUIDIndexedAddModifyTests, self).setUp()
-        indexlist = {"dn": "@INDEXLIST",
-                     "@IDXATTR": [b"x", b"y", b"ou"],
-                     "@IDXONE": [b"1"],
-                     "@IDXGUID": [b"objectUUID"],
-                     "@IDX_DN_GUID": [b"GUID"]}
-        m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
-        self.l.modify(m)
 
 
 class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
@@ -2260,19 +2295,36 @@ class LdbResultTests(LdbBaseTest):
         self.testdir = tempdir()
         self.filename = os.path.join(self.testdir, "test.ldb")
         self.l = ldb.Ldb(self.url(), flags=self.flags())
-        self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
-        self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
-        self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
-        self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1"})
-        self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2"})
-        self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3"})
-        self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4"})
-        self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5"})
-        self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6"})
-        self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7"})
-        self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8"})
-        self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9"})
-        self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10"})
+        try:
+            self.l.add(self.index)
+        except AttributeError:
+            pass
+        self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org",
+                    "objectUUID": b"0123456789abcde0"})
+        self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins",
+                    "objectUUID": b"0123456789abcde1"})
+        self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users",
+                    "objectUUID": b"0123456789abcde2"})
+        self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1",
+                    "objectUUID": b"0123456789abcde3"})
+        self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2",
+                    "objectUUID": b"0123456789abcde4"})
+        self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3",
+                    "objectUUID": b"0123456789abcde5"})
+        self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4",
+                    "objectUUID": b"0123456789abcde6"})
+        self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5",
+                    "objectUUID": b"0123456789abcde7"})
+        self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6",
+                    "objectUUID": b"0123456789abcde8"})
+        self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7",
+                    "objectUUID": b"0123456789abcde9"})
+        self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8",
+                    "objectUUID": b"0123456789abcdea"})
+        self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9",
+                    "objectUUID": b"0123456789abcdeb"})
+        self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10",
+                    "objectUUID": b"0123456789abcdec"})
 
     def tearDown(self):
         shutil.rmtree(self.testdir)
@@ -2373,7 +2425,8 @@ class LdbResultTests(LdbBaseTest):
 
             # write to it
             child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
-                           "name": b"samba.org"})
+                           "name": b"samba.org",
+                           "objectUUID": b"o123456789acbdef"})
 
             os.write(w1, b"added")
 
@@ -2444,7 +2497,8 @@ class LdbResultTests(LdbBaseTest):
 
             # write to it
             child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
-                           "name": b"samba.org"})
+                           "name": b"samba.org",
+                           "objectUUID": b"o123456789acbdef"})
 
             os.write(w1, b"added")
 
@@ -2508,6 +2562,9 @@ class LdbResultTestsLmdb(LdbResultTests):
 
     def setUp(self):
         self.prefix = MDB_PREFIX
+        self.index = {"dn": "@INDEXLIST",
+                      "@IDXGUID": [b"objectUUID"],
+                      "@IDX_DN_GUID": [b"GUID"]}
         super(LdbResultTestsLmdb, self).setUp()
 
     def tearDown(self):
-- 
2.11.0


From 7db22499e0d60fac2e3b0b2fdc0b4c2dc802cf96 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 11/25] ldb_mdb: Apply LMDB key length restrictions at
 key-value layer

We need to enforce the GUID index mode so end-users do not get a supprise
in mid-operation and we enforce a max key length of 511 so that the
index key trunctation is done correctly.

Otherwise the DB will appear to work until a very long key (DN or index)
is used, after which it will be sad.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c     |  22 +++++-
 lib/ldb/ldb_tdb/ldb_tdb.c     |  17 ++++-
 lib/ldb/tests/ldb_lmdb_test.c | 166 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 200 insertions(+), 5 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 711700c2697..f05d1201e05 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -30,6 +30,8 @@
 #define MDB_URL_PREFIX		"mdb://"
 #define MDB_URL_PREFIX_SIZE	(sizeof(MDB_URL_PREFIX)-1)
 
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
 #define MEGABYTE (1024*1024)
 #define GIGABYTE (1024*1024*1024)
 
@@ -641,7 +643,8 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 {
 	int ret;
 	unsigned int mdb_flags;
-
+	int lmdb_max_key_length;
+	
 	if (flags & LDB_FLG_DONT_CREATE_DB) {
 		struct stat st;
 		if (stat(path, &st) != 0) {
@@ -695,6 +698,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 	/* Store the original pid during the LMDB open */
 	lmdb->pid = getpid();
 
+	lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+	/* This will never happen, but if it does make sure to freak out */
+	if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+		talloc_free(lmdb);
+		return ldb_operr(ldb);
+	}
+
 	return LDB_SUCCESS;
 
 }
@@ -746,6 +757,15 @@ int lmdb_connect(struct ldb_context *ldb,
 	if (flags & LDB_FLG_RDONLY) {
 		ltdb->read_only = true;
 	}
+
+	/*
+	 * This maximum length becomes encoded in the index values so
+	 * must never change even if LMDB starts to allow longer keys.
+	 * The override option is max_key_len_for_self_test, and is
+	 * used for testing only.
+	 */
+	ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+	
         return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
 }
 
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1d06566aa30..b1158241ef9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -651,6 +651,15 @@ static int ltdb_add(struct ltdb_context *ctx)
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	int ret = LDB_SUCCESS;
 
+	if (ltdb->max_key_length != 0 && 
+	    ltdb->cache->GUID_index_attribute == NULL &&
+	    !ldb_dn_is_special(req->op.add.message->dn)) {
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				  "Must operate ldb_mdb in GUID "
+				  "index mode, but " LTDB_IDXGUID " not set.");
+			return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+	
 	ret = ltdb_check_special_dn(module, req->op.add.message);
 	if (ret != LDB_SUCCESS) {
 		return ret;
@@ -2149,7 +2158,13 @@ int init_store(struct ltdb_private *ltdb,
 
 	*_module = module;
 	/*
-	 * Set the maximum key length
+	 * Set or override the maximum key length
+	 *
+	 * The ldb_mdb code will have set this to 511, but our tests
+	 * set this even smaller (to make the tests more practical).
+	 *
+	 * This must only be used for the selftest as the length
+	 * becomes encoded in the index keys.
 	 */
 	{
 		const char *len_str =
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index 2902eaf0533..fd82867a275 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -135,12 +135,22 @@ static int ldbtest_setup(void **state)
 {
 	struct ldbtest_ctx *test_ctx;
 	int ret;
+	struct ldb_ldif *ldif;
+	const char *index_ldif =		\
+		"dn: @INDEXLIST\n"
+		"@IDXGUID: objectUUID\n"
+		"@IDX_DN_GUID: GUID\n"
+		"\n";
 
 	ldbtest_noconn_setup((void **) &test_ctx);
 
 	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
 	assert_int_equal(ret, 0);
 
+	while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+		ret = ldb_add(test_ctx->ldb, ldif->msg);
+		assert_int_equal(ret, LDB_SUCCESS);
+	}
 	*state = test_ctx;
 	return 0;
 }
@@ -170,7 +180,8 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	assert_non_null(msg);
 
 	/*
-	 * The zero terminator is part of the key
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
 	 */
 
 	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
@@ -184,8 +195,11 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
-	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+	assert_int_equal(ret, LDB_SUCCESS);
 
 	talloc_free(tmp_ctx);
 }
@@ -207,7 +221,8 @@ static void test_ldb_add_key_len_eq_max(void **state)
 	assert_non_null(msg);
 
 	/*
-	 * The zero terminator is part of the key
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
 	 */
 
 	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
@@ -220,12 +235,145 @@ static void test_ldb_add_key_len_eq_max(void **state)
 	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
 	assert_int_equal(ret, 0);
 
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
 	ret = ldb_add(test_ctx->ldb, msg);
 	assert_int_equal(ret, 0);
 
 	talloc_free(tmp_ctx);
 }
 
+static int ldbtest_setup_noguid(void **state)
+{
+	struct ldbtest_ctx *test_ctx;
+	int ret;
+
+	ldbtest_noconn_setup((void **) &test_ctx);
+
+	ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	*state = test_ctx;
+	return 0;
+}
+
+static void test_ldb_add_special_key_len_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 5;  /* "dn=@" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_special_key_len_eq_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 5;  /* "dn=@" and the zero terminator */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_SUCCESS);
+
+	talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_dn_no_guid_mode(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = LMDB_MAX_KEY_SIZE - 7;  /* "dn=dc=" and the zero terminator */
+	xs_size += 1;                /* want key on char too long        */
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
+
+	talloc_free(tmp_ctx);
+}
+
 int main(int argc, const char **argv)
 {
 	const struct CMUnitTest tests[] = {
@@ -237,6 +385,18 @@ int main(int argc, const char **argv)
 			test_ldb_add_key_len_gt_max,
 			ldbtest_setup,
 			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_special_key_len_eq_max,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_special_key_len_gt_max,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_ldb_add_dn_no_guid_mode,
+			ldbtest_setup_noguid,
+			ldbtest_teardown),
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);
-- 
2.11.0


From 92a33d399bee0db93b9efac2e392a68e60256c97 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 12/25] ldb_mdb: Wrap mdb_env_open

Wrap mdb_env_open to ensure that we only have one MDB_env opened per
database in each process

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 155 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 121 insertions(+), 34 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f05d1201e05..4ab3b3f26d9 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -157,6 +157,9 @@ static int lmdb_store(struct ltdb_private *ltdb,
 	MDB_txn *txn = NULL;
 	MDB_dbi dbi = 0;
 
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
 	if (txn == NULL) {
 		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -212,6 +215,10 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
 	MDB_txn *txn = NULL;
 	MDB_dbi dbi = 0;
 
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
 	if (txn == NULL) {
 		ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -628,73 +635,154 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
 		trans_finished(lmdb, ltx);
 		ltx = lmdb_private_trans_head(lmdb);
 	}
-
-	mdb_env_close(lmdb->env);
 	lmdb->env = NULL;
 
 	return 0;
 }
 
-static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
-				  struct ldb_context *ldb,
-				  const char *path,
-				  unsigned int flags,
-				  struct lmdb_private *lmdb)
+struct mdb_env_wrap {
+	struct mdb_env_wrap *next, *prev;
+	dev_t device;
+	ino_t inode;
+	MDB_env *env;
+	int pid;
+};
+
+static struct mdb_env_wrap *mdb_list;
+
+/* destroy the last connection to an mdb */
+static int mdb_env_wrap_destructor(struct mdb_env_wrap *w)
 {
+	mdb_env_close(w->env);
+	DLIST_REMOVE(mdb_list, w);
+	return 0;
+}
+
+static int lmdb_open_env(TALLOC_CTX *mem_ctx,
+			 MDB_env **env,
+			 struct ldb_context *ldb,
+			 const char *path,
+			 unsigned int flags) {
 	int ret;
-	unsigned int mdb_flags;
-	int lmdb_max_key_length;
-	
-	if (flags & LDB_FLG_DONT_CREATE_DB) {
-		struct stat st;
-		if (stat(path, &st) != 0) {
-			return LDB_ERR_UNAVAILABLE;
+	unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+	/*
+	 * MDB_NOSUBDIR implies there is a separate file called path and a
+	 * separate lockfile called path-lock
+	 */
+
+	struct mdb_env_wrap *w;
+	struct stat st;
+
+	if (stat(path, &st) == 0) {
+		for (w=mdb_list;w;w=w->next) {
+			if (st.st_dev == w->device && st.st_ino == w->inode) {
+				/*
+				 * We must have only one MDB_env per process
+				 */
+				if (!talloc_reference(mem_ctx, w)) {
+					return ldb_oom(ldb);
+				}
+				*env = w->env;
+				return LDB_SUCCESS;
+			}
 		}
 	}
 
-	ret = mdb_env_create(&lmdb->env);
+	w = talloc(mem_ctx, struct mdb_env_wrap);
+	if (w == NULL) {
+		return ldb_oom(ldb);
+	}
+
+	ret = mdb_env_create(env);
 	if (ret != 0) {
 		ldb_asprintf_errstring(
 			ldb,
 			"Could not create MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
-		return LDB_ERR_OPERATIONS_ERROR;
+		return ldb_mdb_err_map(ret);
 	}
 
-	/* Close when lmdb is released */
-	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
-
-	ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+	/*
+	 * Currently we set a 16Gb maximum database size
+	 */
+	ret = mdb_env_set_mapsize(*env, 16LL * GIGABYTE);
 	if (ret != 0) {
 		ldb_asprintf_errstring(
 			ldb,
 			"Could not open MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
+		TALLOC_FREE(w);
 		return ldb_mdb_err_map(ret);
 	}
 
-	mdb_env_set_maxreaders(lmdb->env, 100000);
-	/* MDB_NOSUBDIR implies there is a separate file called path and a
-	 * separate lockfile called path-lock
+	mdb_env_set_maxreaders(*env, 100000);
+	/*
+	 * As we ensure that there is only one MDB_env open per database per
+	 * process. We can not use the MDB_RDONLY flag, as another ldb may be
+	 * opened in read write mode
 	 */
-	mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
-	if (flags & LDB_FLG_RDONLY) {
-		mdb_flags |= MDB_RDONLY;
-	}
 	if (flags & LDB_FLG_NOSYNC) {
 		mdb_flags |= MDB_NOSYNC;
 	}
-	ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+	ret = mdb_env_open(*env, path, mdb_flags, 0644);
 	if (ret != 0) {
 		ldb_asprintf_errstring(ldb,
 				"Could not open DB %s: %s\n",
 				path, mdb_strerror(ret));
-		talloc_free(lmdb);
+		TALLOC_FREE(w);
 		return ldb_mdb_err_map(ret);
 	}
 
+	if (stat(path, &st) != 0) {
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not stat %s:\n",
+			path);
+		TALLOC_FREE(w);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	w->env = *env;
+	w->device = st.st_dev;
+	w->inode  = st.st_ino;
+
+	talloc_set_destructor(w, mdb_env_wrap_destructor);
+
+	DLIST_ADD(mdb_list, w);
+
+	return LDB_SUCCESS;
+
+}
+
+static int lmdb_pvt_open(struct lmdb_private *lmdb,
+			 struct ldb_context *ldb,
+			 const char *path,
+			 unsigned int flags)
+{
+	int ret;
+	int lmdb_max_key_length;
+
+	if (flags & LDB_FLG_DONT_CREATE_DB) {
+		struct stat st;
+		if (stat(path, &st) != 0) {
+			return LDB_ERR_UNAVAILABLE;
+		}
+	}
+
+	ret = lmdb_open_env(lmdb, &lmdb->env, ldb, path, flags);
+	if (ret != 0) {
+		ldb_asprintf_errstring(
+			ldb,
+			"Could not create MDB environment %s: %s\n",
+			path,
+			mdb_strerror(ret));
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	/* Close when lmdb is released */
+	talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
 	/* Store the original pid during the LMDB open */
 	lmdb->pid = getpid();
 
@@ -702,7 +790,6 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
 
 	/* This will never happen, but if it does make sure to freak out */
 	if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
-		talloc_free(lmdb);
 		return ldb_operr(ldb);
 	}
 
@@ -739,17 +826,17 @@ int lmdb_connect(struct ldb_context *ldb,
                 return LDB_ERR_OPERATIONS_ERROR;
         }
 
-	lmdb = talloc_zero(ldb, struct lmdb_private);
+	lmdb = talloc_zero(ltdb, struct lmdb_private);
 	if (lmdb == NULL) {
 		TALLOC_FREE(ltdb);
-                ldb_oom(ldb);
-                return LDB_ERR_OPERATIONS_ERROR;
+                return ldb_oom(ldb);
 	}
 	lmdb->ldb = ldb;
 	ltdb->kv_ops = &lmdb_key_value_ops;
 
-	ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+	ret = lmdb_pvt_open(lmdb, ldb, path, flags);
 	if (ret != LDB_SUCCESS) {
+		TALLOC_FREE(ltdb);
 		return ret;
 	}
 
-- 
2.11.0


From 13d4e31f946b59fc31fd25a7d8b9e5a8abaaece1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 8 Mar 2018 16:47:59 +1300
Subject: [PATCH 13/25] ldb_mdb: Tests for wrap open

Tests to ensure that the mdb_env wrapping code correctly handles
multiple ldb's point to the same physical database file.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 234 ++++++++++++++++++++++++++++++++++++++++
 lib/ldb/wscript                 |   5 +-
 2 files changed, 237 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 6582f37f7d2..1fce9f97882 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -38,6 +38,12 @@
 #define TEST_BE DEFAULT_BE
 #endif /* TEST_BE */
 
+#ifdef TEST_LMDB
+#include "lmdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+#endif
+
 struct ldbtest_ctx {
 	struct tevent_context *ev;
 	struct ldb_context *ldb;
@@ -3820,6 +3826,224 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
 	}
 }
 
+#ifdef TEST_LMDB
+static int test_ldb_multiple_connections_callback(struct ldb_request *req,
+						  struct ldb_reply *ares)
+{
+	int ret;
+	int pipes[2];
+	char buf[2];
+	int pid, child_pid;
+	int wstatus;
+
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
+		break;
+
+	case LDB_REPLY_REFERRAL:
+		return LDB_SUCCESS;
+
+	case LDB_REPLY_DONE:
+		return ldb_request_done(req, LDB_SUCCESS);
+	}
+
+	{
+		/*
+		 * We open a new ldb on an ldb that is already open and
+		 * then close it.
+		 *
+		 * If the multiple connection wrapping is correct the
+		 * underlying MDB_env will be left open and we should see
+		 * an active reader in the child we fork next
+		 */
+		struct ldb_context *ldb = NULL;
+		struct tevent_context *ev = NULL;
+		TALLOC_CTX *mem_ctx = talloc_new(NULL);
+
+		ev = tevent_context_init(mem_ctx);
+		assert_non_null(ev);
+
+		ldb = ldb_init(mem_ctx, ev);
+		assert_non_null(ldb);
+
+		ret = ldb_connect(ldb, TEST_BE"://apitest.ldb" , 0, NULL);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		TALLOC_FREE(ldb);
+		TALLOC_FREE(mem_ctx);
+	}
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct MDB_env *env = NULL;
+		struct MDB_envinfo stat;
+
+		/*
+		 * Check that there are exactly two readers on the MDB file
+		 * backing the ldb.
+		 *
+		 */
+		ret = mdb_env_create(&env);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_create returned (%d)",
+				      ret);
+			exit(ret);
+		}
+
+		ret = mdb_env_open(env,
+				   "apitest.ldb",
+				   MDB_NOSUBDIR | MDB_NOTLS,
+				   0644);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_open returned (%d)",
+				      ret);
+			exit(ret);
+		}
+
+		ret = mdb_env_info(env, &stat);
+		if (ret != 0) {
+			print_error(__location__
+				      " mdb_env_info returned (%d)",
+				      ret);
+			exit(ret);
+		}
+		if (stat.me_numreaders != 2) {
+			print_error(__location__
+				      " Incorrect number of readers (%d)",
+				      stat.me_numreaders);
+			exit(LDB_ERR_CONSTRAINT_VIOLATION);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+	return LDB_SUCCESS;
+
+}
+
+static void test_ldb_close_with_multiple_connections(void **state)
+{
+	struct search_test_ctx *search_test_ctx = NULL;
+	struct ldb_dn *search_dn = NULL;
+	struct ldb_request *req = NULL;
+	int ret = 0;
+
+	search_test_ctx = talloc_get_type_abort(*state, struct search_test_ctx);
+	assert_non_null(search_test_ctx);
+
+	search_dn = ldb_dn_new_fmt(search_test_ctx,
+				   search_test_ctx->ldb_test_ctx->ldb,
+				   "cn=test_search_cn,"
+				   "dc=search_test_entry");
+	assert_non_null(search_dn);
+
+	/*
+	 * The search just needs to call DONE, we don't care about the
+	 * contents of the search for this test
+	 */
+	ret = ldb_build_search_req(&req,
+				   search_test_ctx->ldb_test_ctx->ldb,
+				   search_test_ctx,
+				   search_dn,
+				   LDB_SCOPE_SUBTREE,
+				   "(&(!(filterAttr=*))"
+				   "(cn=test_search_cn))",
+				   NULL,
+				   NULL,
+				   NULL,
+				   test_ldb_multiple_connections_callback,
+				   NULL);
+	assert_int_equal(ret, 0);
+
+	ret = ldb_request(search_test_ctx->ldb_test_ctx->ldb, req);
+	assert_int_equal(ret, 0);
+
+	ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	assert_int_equal(ret, 0);
+}
+
+static struct MDB_env* get_mdb_env(struct ldb_context *ldb)
+{
+	void *data = NULL;
+	struct ltdb_private *ltdb = NULL;
+	struct lmdb_private *lmdb = NULL;
+	struct MDB_env *env = NULL;
+
+	data = ldb_module_get_private(ldb->modules);
+	assert_non_null(data);
+
+	ltdb = talloc_get_type(data, struct ltdb_private);
+	assert_non_null(ltdb);
+
+	lmdb = ltdb->lmdb_private;
+	assert_non_null(lmdb);
+
+	env = lmdb->env;
+	assert_non_null(env);
+
+	return env;
+}
+static void test_multiple_opens(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	struct ldb_context *ldb2 = NULL;
+	struct ldb_context *ldb3 = NULL;
+	struct MDB_env *env1 = NULL;
+	struct MDB_env *env2 = NULL;
+	struct MDB_env *env3 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database again
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb2 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb3 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+	/*
+	 * We now have 3 ldb's open pointing to the same on disk database
+	 * they should all share the same MDB_env
+	 */
+	env1 = get_mdb_env(ldb1);
+	env2 = get_mdb_env(ldb2);
+	env3 = get_mdb_env(ldb3);
+
+	assert_ptr_equal(env1, env2);
+	assert_ptr_equal(env1, env3);
+
+}
+#endif
 
 int main(int argc, const char **argv)
 {
@@ -3984,6 +4208,16 @@ int main(int argc, const char **argv)
 			test_ldb_talloc_destructor_transaction_cleanup,
 			ldbtest_setup,
 			ldbtest_teardown),
+#ifdef TEST_LMDB
+		cmocka_unit_test_setup_teardown(
+			test_ldb_close_with_multiple_connections,
+			ldb_search_test_setup,
+			ldb_search_test_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_multiple_opens,
+			ldbtest_setup,
+			ldbtest_teardown),
+#endif
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index d9f47c798bf..3da73c16d7c 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -443,8 +443,9 @@ def build(bld):
         if bld.CONFIG_SET('HAVE_LMDB'):
             bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
                              source='tests/ldb_mod_op_test.c',
-                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
-                             deps='cmocka ldb',
+                             cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 ' +
+                                    '-DTEST_LMDB=1',
+                             deps='cmocka ldb lmdb',
                              install=False)
             
             bld.SAMBA_BINARY('ldb_lmdb_test',
-- 
2.11.0


From 6b50f5e763444c2917d3d017cb2250cf5afd286d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:14:09 +1300
Subject: [PATCH 14/25] ldb index tests: test large index key value

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_lmdb_test.c | 44 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index fd82867a275..92715fdf96f 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -204,6 +204,46 @@ static void test_ldb_add_key_len_gt_max(void **state)
 	talloc_free(tmp_ctx);
 }
 
+static void test_ldb_add_key_len_2x_gt_max(void **state)
+{
+	int ret;
+	int xs_size = 0;
+	struct ldb_message *msg;
+	struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+							struct ldbtest_ctx);
+	char *xs = NULL;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	msg = ldb_msg_new(tmp_ctx);
+	assert_non_null(msg);
+
+	/*
+	 * The zero terminator is part of the key if we were not in
+	 * GUID mode
+	 */
+
+	xs_size = 2 * LMDB_MAX_KEY_SIZE;
+	xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+	memset(xs, 'x', xs_size);
+
+	msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+	assert_non_null(msg->dn);
+
+	ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+	assert_int_equal(ret, 0);
+
+	ret = ldb_add(test_ctx->ldb, msg);
+	assert_int_equal(ret, LDB_SUCCESS);
+
+	talloc_free(tmp_ctx);
+}
+
 static void test_ldb_add_key_len_eq_max(void **state)
 {
 	int ret;
@@ -386,6 +426,10 @@ int main(int argc, const char **argv)
 			ldbtest_setup,
 			ldbtest_teardown),
 		cmocka_unit_test_setup_teardown(
+			test_ldb_add_key_len_2x_gt_max,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
 			test_ldb_add_special_key_len_eq_max,
 			ldbtest_setup_noguid,
 			ldbtest_teardown),
-- 
2.11.0


From 152b3024e737718693f912ed669e948ecc48e51c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 12:43:25 +1300
Subject: [PATCH 15/25] ldb test: close pipes to stop forked tests failing on
 failure

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_mod_op_test.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 1fce9f97882..c33517134df 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3881,6 +3881,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
 	if (child_pid == 0) {
 		struct MDB_env *env = NULL;
 		struct MDB_envinfo stat;
+		close(pipes[0]);
 
 		/*
 		 * Check that there are exactly two readers on the MDB file
@@ -3929,6 +3930,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
 		}
 		exit(LDB_SUCCESS);
 	}
+	close(pipes[1]);
 	ret = read(pipes[0], buf, 2);
 	assert_int_equal(ret, 2);
 
-- 
2.11.0


From 5271ebf75079bfbb9136bf34142b4d195ed7ec4a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 16/25] ldb_mdb: prevent MDB_env reuse across forks

MDB_env's may not be reused accross forks.  Check the pid that the lmdb
structure was created by, and return an error if it is being used by a
different process.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c       |  33 +++++-
 lib/ldb/tests/ldb_mod_op_test.c | 233 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 264 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 4ab3b3f26d9..eb4609ee98e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -433,6 +433,18 @@ static int lmdb_lock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	struct lmdb_private *lmdb = ltdb->lmdb_private;
+	pid_t pid = getpid();
+
+	if (pid != lmdb->pid) {
+		ldb_asprintf_errstring(
+			lmdb->ldb,
+			__location__": Reusing ldb opend by pid %d in "
+			"process %d\n",
+			lmdb->pid,
+			pid);
+		lmdb->error = MDB_BAD_TXN;
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
 
 	lmdb->error = MDB_SUCCESS;
 	if (ltdb->in_transaction == 0 &&
@@ -472,12 +484,25 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 	struct lmdb_trans *ltx;
 	struct lmdb_trans *ltx_head;
 	MDB_txn *tx_parent;
+	pid_t pid = getpid();
 
 	ltx = talloc_zero(lmdb, struct lmdb_trans);
 	if (ltx == NULL) {
 		return ldb_oom(lmdb->ldb);
 	}
 
+	if (pid != lmdb->pid) {
+		ldb_asprintf_errstring(
+			lmdb->ldb,
+			__location__": Reusing ldb opened by pid %d in "
+			"process %d\n",
+			lmdb->pid,
+			pid);
+		lmdb->error = MDB_BAD_TXN;
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
+
+
 	ltx_head = lmdb_private_trans_head(lmdb);
 
 	tx_parent = lmdb_trans_get_tx(ltx_head);
@@ -645,7 +670,7 @@ struct mdb_env_wrap {
 	dev_t device;
 	ino_t inode;
 	MDB_env *env;
-	int pid;
+	pid_t pid;
 };
 
 static struct mdb_env_wrap *mdb_list;
@@ -672,10 +697,13 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
 
 	struct mdb_env_wrap *w;
 	struct stat st;
+	pid_t pid = getpid();
 
 	if (stat(path, &st) == 0) {
 		for (w=mdb_list;w;w=w->next) {
-			if (st.st_dev == w->device && st.st_ino == w->inode) {
+			if (st.st_dev == w->device &&
+			    st.st_ino == w->inode &&
+			    pid == w->pid) {
 				/*
 				 * We must have only one MDB_env per process
 				 */
@@ -746,6 +774,7 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
 	w->env = *env;
 	w->device = st.st_dev;
 	w->inode  = st.st_ino;
+	w->pid = pid;
 
 	talloc_set_destructor(w, mdb_env_wrap_destructor);
 
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index c33517134df..da20485e016 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -4045,6 +4045,227 @@ static void test_multiple_opens(void **state)
 	assert_ptr_equal(env1, env3);
 
 }
+
+static void test_multiple_opens_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	struct ldb_context *ldb2 = NULL;
+	struct MDB_env *env1 = NULL;
+	struct MDB_env *env2 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database again
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	ldb2 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+	assert_int_equal(ret, 0);
+
+	env1 = get_mdb_env(ldb1);
+	env2 = get_mdb_env(ldb2);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct ldb_context *ldb3 = NULL;
+		struct MDB_env *env3 = NULL;
+
+		close(pipes[0]);
+		ldb3 = ldb_init(test_ctx, test_ctx->ev);
+		ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+		if (ret != 0) {
+			print_error(__location__": ldb_connect returned (%d)\n",
+				    ret);
+			exit(ret);
+		}
+		env3 = get_mdb_env(ldb3);
+		if (env1 != env2) {
+			print_error(__location__": env1 != env2\n");
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		if (env1 == env3) {
+			print_error(__location__": env1 == env3\n");
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_transaction_start_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		close(pipes[0]);
+		ret = ldb_transaction_start(ldb1);
+		if (ret != LDB_ERR_PROTOCOL_ERROR) {
+			print_error(__location__": ldb_transaction_start "
+				    "returned (%d) %s\n",
+				    ret,
+				    ldb1->err_string);
+			exit(LDB_ERR_OTHER);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_lock_read_across_fork(void **state)
+{
+	struct ldb_context *ldb1 = NULL;
+	int ret;
+	struct ldbtest_ctx *test_ctx = NULL;
+	int pipes[2];
+	char buf[2];
+	int wstatus;
+	pid_t pid, child_pid;
+
+	test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+	/*
+	 * Open the database
+	 */
+	ldb1 = ldb_init(test_ctx, test_ctx->ev);
+	ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+	assert_int_equal(ret, 0);
+
+	ret = pipe(pipes);
+	assert_int_equal(ret, 0);
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		struct ldb_dn *basedn;
+		struct ldb_result *result = NULL;
+
+		close(pipes[0]);
+
+		basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+		assert_non_null(basedn);
+
+		ret = ldb_search(test_ctx->ldb,
+				 test_ctx,
+				 &result,
+				 basedn,
+				 LDB_SCOPE_BASE,
+				 NULL,
+				 NULL);
+		if (ret != LDB_ERR_PROTOCOL_ERROR) {
+			print_error(__location__": ldb_search "
+				    "returned (%d) %s\n",
+				    ret,
+				    ldb1->err_string);
+			exit(LDB_ERR_OTHER);
+		}
+
+		ret = write(pipes[1], "GO", 2);
+		if (ret != 2) {
+			print_error(__location__
+				      " write returned (%d)",
+				      ret);
+			exit(LDB_ERR_OPERATIONS_ERROR);
+		}
+		exit(LDB_SUCCESS);
+	}
+	close(pipes[1]);
+	ret = read(pipes[0], buf, 2);
+	assert_int_equal(ret, 2);
+
+	pid = waitpid(child_pid, &wstatus, 0);
+	assert_int_equal(pid, child_pid);
+
+	assert_true(WIFEXITED(wstatus));
+
+	assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+	{
+		/*
+		 * Ensure that the search actually succeeds on the opening
+		 * pid
+		 */
+		struct ldb_dn *basedn;
+		struct ldb_result *result = NULL;
+
+		close(pipes[0]);
+
+		basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+		assert_non_null(basedn);
+
+		ret = ldb_search(test_ctx->ldb,
+				 test_ctx,
+				 &result,
+				 basedn,
+				 LDB_SCOPE_BASE,
+				 NULL,
+				 NULL);
+		assert_int_equal(0, ret);
+	}
+}
 #endif
 
 int main(int argc, const char **argv)
@@ -4219,6 +4440,18 @@ int main(int argc, const char **argv)
 			test_multiple_opens,
 			ldbtest_setup,
 			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_multiple_opens_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_transaction_start_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
+		cmocka_unit_test_setup_teardown(
+			test_lock_read_across_fork,
+			ldbtest_setup,
+			ldbtest_teardown),
 #endif
 	};
 
-- 
2.11.0


From acb7db2c25988d2405095bf8738588efaf32647a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:25:28 +1300
Subject: [PATCH 17/25] ldb: make backends expose if there is an active
 transaction

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 37 +++++++++++++++++++++----------------
 lib/ldb/ldb_tdb/ldb_tdb.c |  5 +++++
 lib/ldb/ldb_tdb/ldb_tdb.h |  1 +
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index eb4609ee98e..bb398ba89d5 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -91,6 +91,11 @@ static int lmdb_error_at(struct ldb_context *ldb,
 	return ldb_err;
 }
 
+
+static bool lmdb_transaction_active(struct ltdb_private *ltdb) {
+	return ltdb->lmdb_private->txlist != NULL;
+}
+
 static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
 {
 	if (ltx == NULL) {
@@ -581,23 +586,23 @@ static bool lmdb_changed(struct ltdb_private *ltdb)
 	return true;
 }
 
-
 static struct kv_db_ops lmdb_key_value_ops = {
-	.store             = lmdb_store,
-	.delete            = lmdb_delete,
-	.iterate           = lmdb_traverse_fn,
-	.update_in_iterate = lmdb_update_in_iterate,
-	.fetch_and_parse   = lmdb_parse_record,
-	.lock_read         = lmdb_lock_read,
-	.unlock_read       = lmdb_unlock_read,
-	.begin_write       = lmdb_transaction_start,
-	.prepare_write     = lmdb_transaction_prepare_commit,
-	.finish_write      = lmdb_transaction_commit,
-	.abort_write       = lmdb_transaction_cancel,
-	.error             = lmdb_error,
-	.errorstr          = lmdb_errorstr,
-	.name              = lmdb_name,
-	.has_changed       = lmdb_changed,
+	.store              = lmdb_store,
+	.delete             = lmdb_delete,
+	.iterate            = lmdb_traverse_fn,
+	.update_in_iterate  = lmdb_update_in_iterate,
+	.fetch_and_parse    = lmdb_parse_record,
+	.lock_read          = lmdb_lock_read,
+	.unlock_read        = lmdb_unlock_read,
+	.begin_write        = lmdb_transaction_start,
+	.prepare_write      = lmdb_transaction_prepare_commit,
+	.finish_write       = lmdb_transaction_commit,
+	.abort_write        = lmdb_transaction_cancel,
+	.error              = lmdb_error,
+	.errorstr           = lmdb_errorstr,
+	.name               = lmdb_name,
+	.has_changed        = lmdb_changed,
+	.transaction_active = lmdb_transaction_active,
 };
 
 static const char *lmdb_get_path(const char *url)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index b1158241ef9..66cd6cb9ff7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1924,6 +1924,10 @@ static bool ltdb_tdb_changed(struct ltdb_private *ltdb)
 	return has_changed;
 }
 
+static bool ltdb_transaction_active(struct ltdb_private *ltdb) {
+	return tdb_transaction_active(ltdb->tdb);
+}
+
 static const struct kv_db_ops key_value_ops = {
 	.store = ltdb_tdb_store,
 	.delete = ltdb_tdb_delete,
@@ -1940,6 +1944,7 @@ static const struct kv_db_ops key_value_ops = {
 	.errorstr = ltdb_errorstr,
 	.name = ltdb_tdb_name,
 	.has_changed = ltdb_tdb_changed,
+	.transaction_active = ltdb_transaction_active,
 };
 
 static void ltdb_callback(struct tevent_context *ev,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 7d92804a8e9..59fd06aac40 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -29,6 +29,7 @@ struct kv_db_ops {
 	const char * (*errorstr)(struct ltdb_private *ltdb);
 	const char * (*name)(struct ltdb_private *ltdb);
 	bool (*has_changed)(struct ltdb_private *ltdb);
+	bool (*transaction_active)(struct ltdb_private *ltdb);
 };
 
 /* this private structure is used by the ltdb backend in the
-- 
2.11.0


From e51188693295c1acdb6df272735297a7b7f4be45 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:27:10 +1300
Subject: [PATCH 18/25] ldb_tdb: Do not make search or DB modifications without
 a lock

The ldb_cache startup code would previously not take a read lock
nor a sufficiently wide write transaction.

The new code takes a read lock, and if it needs to write takes a
write lock (transaction) and re-reads before continuing.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_cache.c | 50 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 4790bcd7e53..0d16452b913 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -386,6 +386,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	uint64_t seq;
 	struct ldb_message *baseinfo = NULL, *options = NULL;
 	const struct ldb_schema_attribute *a;
+	bool have_write_txn = false;
 	int r;
 
 	ldb = ldb_module_get_ctx(module);
@@ -406,29 +407,38 @@ int ltdb_cache_load(struct ldb_module *module)
 	baseinfo_dn = ldb_dn_new(baseinfo, ldb, LTDB_BASEINFO);
 	if (baseinfo_dn == NULL) goto failed;
 
+	r = ltdb->kv_ops->lock_read(module);
+	if (r != LDB_SUCCESS) {
+		goto failed;
+	}
 	r= ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0);
 	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
-		goto failed;
+		goto failed_and_unlock;
 	}
-	
+
 	/* possibly initialise the baseinfo */
 	if (r == LDB_ERR_NO_SUCH_OBJECT) {
 
+		/* Give up the read lock, try again with a write lock */
+		r = ltdb->kv_ops->unlock_read(module);
+		if (r != LDB_SUCCESS) {
+			goto failed;
+		}
+		
 		if (ltdb->kv_ops->begin_write(ltdb) != 0) {
 			goto failed;
 		}
 
+		have_write_txn = true;
+		
 		/* error handling for ltdb_baseinfo_init() is by
 		   looking for the record again. */
 		ltdb_baseinfo_init(module);
 
-		if (ltdb->kv_ops->finish_write(ltdb) != 0) {
-			goto failed;
-		}
-
 		if (ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0) != LDB_SUCCESS) {
-			goto failed;
+			goto failed_and_unlock;
 		}
+
 	}
 
 	/* Ignore the result, and update the sequence number */
@@ -443,16 +453,17 @@ int ltdb_cache_load(struct ldb_module *module)
 	ltdb->sequence_number = seq;
 
 	/* Read an interpret database options */
+	
 	options = ldb_msg_new(ltdb->cache);
-	if (options == NULL) goto failed;
+	if (options == NULL) goto failed_and_unlock;
 
 	options_dn = ldb_dn_new(options, ldb, LTDB_OPTIONS);
-	if (options_dn == NULL) goto failed;
+	if (options_dn == NULL) goto failed_and_unlock;
 
 	r= ltdb_search_dn1(module, options_dn, options, 0);
 	talloc_free(options_dn);
 	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 	
 	/* set flags if they do exist */
@@ -479,7 +490,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	ltdb_attributes_unload(module);
 
 	if (ltdb_index_load(module, ltdb) == -1) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 
 	/*
@@ -488,7 +499,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	 * partition module.
 	 */
 	if (ltdb_attributes_load(module) == -1) {
-		goto failed;
+		goto failed_and_unlock;
 	}
 
 	ltdb->GUID_index_syntax = NULL;
@@ -503,10 +514,25 @@ int ltdb_cache_load(struct ldb_module *module)
 	}
 
 done:
+	if (have_write_txn) {
+		if (ltdb->kv_ops->finish_write(ltdb) != 0) {
+			goto failed;
+		}
+	} else {
+		ltdb->kv_ops->unlock_read(module);
+	}
+	
 	talloc_free(options);
 	talloc_free(baseinfo);
 	return 0;
 
+failed_and_unlock:
+	if (have_write_txn) {
+		ltdb->kv_ops->abort_write(ltdb);
+	} else {
+		ltdb->kv_ops->unlock_read(module);
+	}
+	
 failed:
 	talloc_free(options);
 	talloc_free(baseinfo);
-- 
2.11.0


From 109de608e9f5687ceb72911106f0804e2d9a8dc3 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:29:25 +1300
Subject: [PATCH 19/25] ldb_mdb: Remove implicit read lock and remove
 transaction counter

The way to know if we are in a transaction is if there is a non-NULL
transaction handle.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 51 +++++++++++++++--------------------------------
 1 file changed, 16 insertions(+), 35 deletions(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index bb398ba89d5..3c2dc980349 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -129,26 +129,21 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
 	return ltx;
 }
 
+
 static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
 {
 	MDB_txn *txn;
-	if (lmdb->read_txn != NULL) {
-		return lmdb->read_txn;
-	}
 
 	txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
-	if (txn == NULL) {
-		int ret;
-		ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
-		if (ret != 0) {
-			lmdb->error = ret;
-			ldb_asprintf_errstring(lmdb->ldb,
-					       "%s failed: %s\n", __FUNCTION__,
-					       mdb_strerror(ret));
-		}
-		lmdb->read_txn = txn;
+	if (txn != NULL) {
+		return txn;
+	}
+	if (lmdb->read_txn != NULL) {
+		return lmdb->read_txn;
 	}
-	return txn;
+	lmdb->error = MDB_BAD_TXN;
+	ldb_set_errstring(lmdb->ldb, __location__":No active transaction\n");
+	return NULL;
 }
 
 static int lmdb_store(struct ltdb_private *ltdb,
@@ -193,10 +188,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
 		MDB_val value;
 		lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
 		if (lmdb->error != MDB_SUCCESS) {
-			if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-				mdb_txn_commit(lmdb->read_txn);
-				lmdb->read_txn = NULL;
-			}
 			return ldb_mdb_error(lmdb->ldb, lmdb->error);
 		}
 		mdb_flags = 0;
@@ -301,11 +292,6 @@ done:
 		mdb_cursor_close(cursor);
 	}
 
-	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-		mdb_txn_commit(lmdb->read_txn);
-		lmdb->read_txn = NULL;
-	}
-
 	if (lmdb->error != MDB_SUCCESS) {
 		return ldb_mdb_error(lmdb->ldb, lmdb->error);
 	}
@@ -409,10 +395,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
 	if (lmdb->error != MDB_SUCCESS) {
 		/* TODO closing a handle should not even be necessary */
 		mdb_dbi_close(lmdb->env, dbi);
-		if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-			mdb_txn_commit(lmdb->read_txn);
-			lmdb->read_txn = NULL;
-		}
 		if (lmdb->error == MDB_NOTFOUND) {
 			return LDB_ERR_NO_SUCH_OBJECT;
 		}
@@ -424,11 +406,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
 	/* TODO closing a handle should not even be necessary */
 	mdb_dbi_close(lmdb->env, dbi);
 
-	/* We created a read transaction, commit it */
-	if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
-		mdb_txn_commit(lmdb->read_txn);
-		lmdb->read_txn = NULL;
-	}
 	return parser(key, data, ctx);
 }
 
@@ -452,7 +429,7 @@ static int lmdb_lock_read(struct ldb_module *module)
 	}
 
 	lmdb->error = MDB_SUCCESS;
-	if (ltdb->in_transaction == 0 &&
+	if (lmdb_transaction_active(ltdb) == false &&
 	    ltdb->read_lock_count == 0) {
 		lmdb->error = mdb_txn_begin(lmdb->env,
 					    NULL,
@@ -472,7 +449,7 @@ static int lmdb_unlock_read(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+	if (lmdb_transaction_active(ltdb) == false && ltdb->read_lock_count == 1) {
 		struct lmdb_private *lmdb = ltdb->lmdb_private;
 		mdb_txn_commit(lmdb->read_txn);
 		lmdb->read_txn = NULL;
@@ -491,6 +468,11 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 	MDB_txn *tx_parent;
 	pid_t pid = getpid();
 
+	/* Do not take out the transaction lock on a read-only DB */
+	if (ltdb->read_only) {
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+	
 	ltx = talloc_zero(lmdb, struct lmdb_trans);
 	if (ltx == NULL) {
 		return ldb_oom(lmdb->ldb);
@@ -507,7 +489,6 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
 		return LDB_ERR_PROTOCOL_ERROR;
 	}
 
-
 	ltx_head = lmdb_private_trans_head(lmdb);
 
 	tx_parent = lmdb_trans_get_tx(ltx_head);
-- 
2.11.0


From e41a651aa74d4a2fa9e5da4d01da0ea336b06b44 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:28:18 +1300
Subject: [PATCH 20/25] ldb_tdb: Disallow TDB nested transactions and use
 tdb_transaction_active()

This avoids keeping a counter, which can be error-prone.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 22 ++++++++++------------
 lib/ldb/ldb_tdb/ldb_tdb.h |  1 -
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 66cd6cb9ff7..babee775008 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -101,7 +101,7 @@ static int ltdb_lock_read(struct ldb_module *module)
 	int tdb_ret = 0;
 	int ret;
 
-	if (ltdb->in_transaction == 0 &&
+	if (tdb_transaction_active(ltdb->tdb) == false &&
 	    ltdb->read_lock_count == 0) {
 		tdb_ret = tdb_lockall_read(ltdb->tdb);
 	}
@@ -128,7 +128,7 @@ static int ltdb_unlock_read(struct ldb_module *module)
 {
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
-	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+	if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
 		tdb_unlockall_read(ltdb->tdb);
 		ltdb->read_lock_count--;
 		return 0;
@@ -379,7 +379,7 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
 
 	/* only allow modifies inside a transaction, otherwise the
 	 * ldb is unsafe */
-	if (ltdb->in_transaction == 0) {
+	if (ltdb->kv_ops->transaction_active(ltdb) == false) {
 		ldb_set_errstring(ldb_module_get_ctx(module), "ltdb modify without transaction");
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
@@ -1478,7 +1478,6 @@ static int ltdb_start_trans(struct ldb_module *module)
 		return ltdb->kv_ops->error(ltdb);
 	}
 
-	ltdb->in_transaction++;
 
 	ltdb_index_transaction_start(module);
 
@@ -1499,8 +1498,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	if (ltdb->in_transaction != 1) {
-		return LDB_SUCCESS;
+	if (!ltdb->kv_ops->transaction_active(ltdb)) {
+		ldb_set_errstring(ldb_module_get_ctx(module),
+				  "ltdb_prepare_commit() called "
+				  "without transaction active");
+		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
 	/*
@@ -1524,13 +1526,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 	ret = ltdb_index_transaction_commit(module);
 	if (ret != LDB_SUCCESS) {
 		ltdb->kv_ops->abort_write(ltdb);
-		ltdb->in_transaction--;
 		return ret;
 	}
 
 	if (ltdb->kv_ops->prepare_write(ltdb) != 0) {
 		ret = ltdb->kv_ops->error(ltdb);
-		ltdb->in_transaction--;
 		ldb_debug_set(ldb_module_get_ctx(module),
 			      LDB_DEBUG_FATAL,
 			      "Failure during "
@@ -1558,7 +1558,6 @@ static int ltdb_end_trans(struct ldb_module *module)
 		}
 	}
 
-	ltdb->in_transaction--;
 	ltdb->prepared_commit = false;
 
 	if (ltdb->kv_ops->finish_write(ltdb) != 0) {
@@ -1578,7 +1577,6 @@ static int ltdb_del_trans(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
-	ltdb->in_transaction--;
 
 	if (ltdb_index_transaction_cancel(module) != 0) {
 		ltdb->kv_ops->abort_write(ltdb);
@@ -1808,7 +1806,7 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
 		.ctx = ctx,
 		.ltdb = ltdb
 	};
-	if (ltdb->in_transaction != 0) {
+	if (tdb_transaction_active(ltdb->tdb)) {
 		return tdb_traverse(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
 	} else {
 		return tdb_traverse_read(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
@@ -2213,7 +2211,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
 		path = url;
 	}
 
-	tdb_flags = TDB_DEFAULT | TDB_SEQNUM;
+	tdb_flags = TDB_DEFAULT | TDB_SEQNUM | TDB_DISALLOW_NESTING;
 
 	/* check for the 'nosync' option */
 	if (flags & LDB_FLG_NOSYNC) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 59fd06aac40..9c3f8d89d8d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -54,7 +54,6 @@ struct ltdb_private {
 		const char *GUID_index_dn_component;
 	} *cache;
 
-	int in_transaction;
 
 	bool check_base;
 	bool disallow_dn_filter;
-- 
2.11.0


From ec064f732c725b0c775229df17000eb23bb24c03 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 4 Apr 2018 17:21:30 +1200
Subject: [PATCH 21/25] ldb_tdb: Disallow reads without a transaction or read
 lock

This will ensure we match LMDB behaviour and avoid a repeat of the per-record locking
issues (compared with full DB locking) we had before Samba 4.7.

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index babee775008..4f31210c5fc 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1899,6 +1899,11 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
 	};
 	int ret;
 
+	if (tdb_transaction_active(ltdb->tdb) == false &&
+	    ltdb->read_lock_count == 0) {
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
+
 	ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
 			       &kv_ctx);
 	if (ret == 0) {
-- 
2.11.0


From 27bed008d1712acb092e05272ea088e34a88b5a9 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:14:10 +1300
Subject: [PATCH 22/25] ldb tests: api ensure database correctly populated

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/python/api.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index bb2c918e9f6..1222d122199 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -1316,6 +1316,19 @@ class AddModifyTests(LdbBaseTest):
                     "name": b"Admins",
                     "x": "z", "y": "a",
                     "objectUUID": b"0123456789abcde2"})
+
+        res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+                             scope=ldb.SCOPE_SUBTREE,
+                             expression="(objectUUID=0123456789abcde1)")
+        self.assertEqual(len(res2), 1)
+        self.assertEqual(str(res2[0].dn), "OU=DUP,DC=SAMBA,DC=ORG")
+
+        res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+                             scope=ldb.SCOPE_SUBTREE,
+                             expression="(objectUUID=0123456789abcde2)")
+        self.assertEqual(len(res3), 1)
+        self.assertEqual(str(res3[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
         try:
             self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
                           "OU=DUP2,DC=SAMBA,DC=ORG")
-- 
2.11.0


From 5d4a9692183f002a9ccf542f65b284d3773b918f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:15:12 +1300
Subject: [PATCH 23/25] ldb tests: add cmocka tests of kv operation
 interactions with transactions

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/tests/ldb_kv_ops_test.c | 184 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 184 insertions(+)

diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
index 4cfb618ea94..501d1f1599b 100644
--- a/lib/ldb/tests/ldb_kv_ops_test.c
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -270,6 +270,63 @@ static void test_add_get(void **state)
 }
 
 /*
+ * Test that attempts to read data without a read transaction fail.
+ */
+static void test_read_outside_transaction(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	uint8_t key_val[] = "TheKey";
+	struct ldb_val key = {
+		.data   = key_val,
+		.length = sizeof(key_val)
+	};
+
+	uint8_t value[] = "The record contents";
+	struct ldb_val data = {
+		.data    = value,
+		.length = sizeof(value)
+	};
+
+	struct ldb_val read;
+
+	int flags = 0;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the record
+	 */
+	ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * And now read it back
+	 * Note there is no read transaction active
+	 */
+	ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+	assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
  * Test that data can be deleted from the kv store
  */
 static void test_delete(void **state)
@@ -731,6 +788,125 @@ static void test_iterate(void **state)
 	TALLOC_FREE(tmp_ctx);
 }
 
+struct update_context {
+	struct ldb_context* ldb;
+	int visits[NUM_RECS];
+};
+
+static int update_fn(struct ltdb_private *ltdb,
+		     struct ldb_val key,
+		     struct ldb_val data,
+		     void *ctx) {
+
+	struct ldb_val new_key;
+	struct ldb_module *module = NULL;
+	struct update_context *context =NULL;
+	int ret = LDB_SUCCESS;
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(ltdb);
+	assert_non_null(tmp_ctx);
+
+	context = talloc_get_type_abort(ctx, struct update_context);
+
+	module = talloc_zero(tmp_ctx, struct ldb_module);
+	module->ldb = context->ldb;
+
+	if (strncmp("key ", (char *) key.data, 4) == 0) {
+		int i = strtol((char *) &key.data[4], NULL, 10);
+		context->visits[i]++;
+		new_key.data = talloc_memdup(tmp_ctx, key.data, key.length);
+		new_key.length  = key.length;
+		new_key.data[0] = 'K';
+
+		ret = ltdb->kv_ops->update_in_iterate(ltdb,
+						      key,
+						      new_key,
+						      data,
+						      &module);
+	}
+	TALLOC_FREE(tmp_ctx);
+	return ret;
+}
+
+/*
+ * Test that update_in_iterate behaves as expected.
+ */
+static void test_update_in_iterate(void **state)
+{
+	int ret;
+	struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+							  struct test_ctx);
+	struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+	int i;
+	struct update_context *context = NULL;
+
+
+	TALLOC_CTX *tmp_ctx;
+
+	tmp_ctx = talloc_new(test_ctx);
+	assert_non_null(tmp_ctx);
+
+	context = talloc_zero(tmp_ctx, struct update_context);
+	assert_non_null(context);
+	context->ldb = test_ctx->ldb;
+	/*
+	 * Begin a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Write the records
+	 */
+	for (i = 0; i < NUM_RECS; i++) {
+		struct ldb_val key;
+		struct ldb_val rec;
+		int flags = 0;
+
+		key.data   = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+		key.length = strlen((char *)key.data) + 1;
+
+		rec.data   = (uint8_t *) talloc_asprintf(tmp_ctx,
+							 "data for record (%04d)",
+							 i);
+		rec.length = strlen((char *)rec.data) + 1;
+
+		ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+		assert_int_equal(ret, 0);
+
+		TALLOC_FREE(key.data);
+		TALLOC_FREE(rec.data);
+	}
+
+	/*
+	 * Commit the transaction
+	 */
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	/*
+	 * Now iterate over the kv store and ensure that all the
+	 * records are visited.
+	 */
+
+	/*
+	 * Needs to be done inside a transaction
+	 */
+	ret = ltdb->kv_ops->begin_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	ret = ltdb->kv_ops->iterate(ltdb, update_fn, context);
+	for (i = 0; i < NUM_RECS; i++) {
+		assert_int_equal(1, context->visits[i]);
+	}
+
+	ret = ltdb->kv_ops->finish_write(ltdb);
+	assert_int_equal(ret, 0);
+
+	TALLOC_FREE(tmp_ctx);
+}
+
 /*
  * Ensure that writes are not visible until the transaction has been
  * committed.
@@ -1371,6 +1547,10 @@ int main(int argc, const char **argv)
 			setup,
 			teardown),
 		cmocka_unit_test_setup_teardown(
+			test_read_outside_transaction,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
 			test_write_outside_transaction,
 			setup,
 			teardown),
@@ -1383,6 +1563,10 @@ int main(int argc, const char **argv)
 			setup,
 			teardown),
 		cmocka_unit_test_setup_teardown(
+			test_update_in_iterate,
+			setup,
+			teardown),
+		cmocka_unit_test_setup_teardown(
 			test_write_transaction_isolation,
 			setup,
 			teardown),
-- 
2.11.0


From 94a5d21849ecce88de143b1fb39895df1faefa67 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 21 Mar 2018 11:38:22 +1300
Subject: [PATCH 24/25] ldb_mdb: handle EBADE from mdb_env_open

Under some circumstances mdb_env_open returns EBADE, we treat this as
indicating the file is not a valid lmdb format file.

Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ldb_mdb/ldb_mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 3c2dc980349..f607afee648 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -42,6 +42,7 @@ int ldb_mdb_err_map(int lmdb_err)
 		return LDB_SUCCESS;
 	case EIO:
 		return LDB_ERR_OPERATIONS_ERROR;
+	case EBADE:
 	case MDB_INCOMPATIBLE:
 	case MDB_CORRUPTED:
 	case MDB_INVALID:
@@ -792,7 +793,7 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
 			"Could not create MDB environment %s: %s\n",
 			path,
 			mdb_strerror(ret));
-		return LDB_ERR_OPERATIONS_ERROR;
+		return ldb_mdb_err_map(ret);
 	}
 
 	/* Close when lmdb is released */
-- 
2.11.0


From 3ddec60fd0db9d8500bc28630b8b85bfffe3abc2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 20 Mar 2018 12:58:02 +1300
Subject: [PATCH 25/25] ldb: Release ldb 1.4.0

* New LMDB backend
* Comprehensive tests for index behaviour
* Enforce transactions for writes
* Enforce read lock use for all reads

Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
 lib/ldb/ABI/ldb-1.4.0.sigs            | 279 ++++++++++++++++++++++++++++++++++
 lib/ldb/ABI/pyldb-util-1.4.0.sigs     |   2 +
 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs |   2 +
 lib/ldb/wscript                       |   2 +-
 4 files changed, 284 insertions(+), 1 deletion(-)
 create mode 100644 lib/ldb/ABI/ldb-1.4.0.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util-1.4.0.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs

diff --git a/lib/ldb/ABI/ldb-1.4.0.sigs b/lib/ldb/ABI/ldb-1.4.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.4.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.4.0.sigs b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 3da73c16d7c..240179f5563 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 APPNAME = 'ldb'
-VERSION = '1.3.2'
+VERSION = '1.4.0'
 
 blddir = 'bin'
 
-- 
2.11.0



More information about the samba-technical mailing list