[PATCH][WIP] LMDB full patch set
Andrew Bartlett
abartlet at samba.org
Wed Apr 11 11:24:05 UTC 2018
On Wed, 2018-04-11 at 19:32 +1200, Andrew Bartlett wrote:
> G'Day metze,
>
> As you may have noticed I'm chasing down some existing TDB locking
> issues that this patch seems to have made more common.
>
> However the patch set is essentially final and the LMDB parts haven't
> changed since those mailed to you this time last week.
>
> Here is the current status:
> https://gitlab.com/catalyst-samba/samba/commits/abartlet-lmdb-reordered
> -2
I think I've fixed the last bug (or at least the last one for tonight)
and here is the CI for the updated branch above (and attached):
https://gitlab.com/catalyst-samba/samba/pipelines/20322095
> I know you are busy, so I'm assuming this has dropped off your radar.
> That's OK, but I'm writing this to say that once I get past the CI,
> deal with the knownfail Jim requested and obtain a second review on the
> amended branch, I'm looking to push tomorrow or Friday.
>
> If you have any comments please get them to me for my work day
> tomorrow.
>
> Thanks,
>
> Andrew Bartlett
>
--
Andrew Bartlett http://samba.org/~abartlet/
Authentication Developer, Samba Team http://samba.org
Samba Developer, Catalyst IT http://catalyst.net.nz/services/samba
-------------- next part --------------
From 9fa7d7b28be0bca9599b1299950f17cdb704b4f0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 11 Apr 2018 22:49:31 +1200
Subject: [PATCH 01/42] dsdb: Check for userPassword support after loading the
databases
The net result of this is only that userPassword values (which were
world readable when set) would still be visible after userPassword
started setting the main DB password.
In AD, those values become hidden once the dSHeuristics bit is set,
but Samba lost that when fixing a performance issue with
f26a2845bd42e580ddeaf0eecc9b46b823a0c6bc
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13378
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
selftest/knownfail.d/dsheuristics_userPassword | 1 +
source4/dsdb/samdb/ldb_modules/acl.c | 18 +++++++++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
create mode 100644 selftest/knownfail.d/dsheuristics_userPassword
diff --git a/selftest/knownfail.d/dsheuristics_userPassword b/selftest/knownfail.d/dsheuristics_userPassword
new file mode 100644
index 00000000000..6981255d9e9
--- /dev/null
+++ b/selftest/knownfail.d/dsheuristics_userPassword
@@ -0,0 +1 @@
+^samba4.ldap.passwords.python\(.*\).__main__.PasswordTests.test_modify_dsheuristics_userPassword
diff --git a/source4/dsdb/samdb/ldb_modules/acl.c b/source4/dsdb/samdb/ldb_modules/acl.c
index d750362c47f..8b1dcbeed51 100644
--- a/source4/dsdb/samdb/ldb_modules/acl.c
+++ b/source4/dsdb/samdb/ldb_modules/acl.c
@@ -108,8 +108,6 @@ static int acl_module_init(struct ldb_module *module)
NULL, "acl", "search", true);
ldb_module_set_private(module, data);
- data->userPassword_support = dsdb_user_password_support(module, module, NULL);
-
mem_ctx = talloc_new(module);
if (!mem_ctx) {
return ldb_oom(ldb);
@@ -180,7 +178,21 @@ static int acl_module_init(struct ldb_module *module)
done:
talloc_free(mem_ctx);
- return ldb_next_init(module);
+ ret = ldb_next_init(module);
+
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
+ /*
+ * Check this after the modules have be initalised so we
+ * can actually read the backend DB.
+ */
+ data->userPassword_support
+ = dsdb_user_password_support(module,
+ module,
+ NULL);
+ return ret;
}
static int acl_allowedAttributes(struct ldb_module *module,
--
2.14.3
From 420074e5a5ff44bcb6e93998f2434f97227f9a5f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 11 Apr 2018 22:47:03 +1200
Subject: [PATCH 02/42] dsdb: check for dSHeuristics more carefully
This check would pass if the dSHeuristics was treated as always being
000000000 for searches which is not enough, we must check for a value
of 000000001 (userPassword enabled).
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13378
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
selftest/knownfail.d/dsheuristics_userPassword | 1 -
source4/dsdb/tests/python/passwords.py | 31 +++++++++++++++++++++-----
2 files changed, 25 insertions(+), 7 deletions(-)
delete mode 100644 selftest/knownfail.d/dsheuristics_userPassword
diff --git a/selftest/knownfail.d/dsheuristics_userPassword b/selftest/knownfail.d/dsheuristics_userPassword
deleted file mode 100644
index 6981255d9e9..00000000000
--- a/selftest/knownfail.d/dsheuristics_userPassword
+++ /dev/null
@@ -1 +0,0 @@
-^samba4.ldap.passwords.python\(.*\).__main__.PasswordTests.test_modify_dsheuristics_userPassword
diff --git a/source4/dsdb/tests/python/passwords.py b/source4/dsdb/tests/python/passwords.py
index c8c2b762a64..bbb8be1d2ca 100755
--- a/source4/dsdb/tests/python/passwords.py
+++ b/source4/dsdb/tests/python/passwords.py
@@ -985,7 +985,8 @@ userPassword: thatsAcomplPASS4
res = ldb1.search("cn=testuser,cn=users," + self.base_dn,
scope=SCOPE_BASE, attrs=["userPassword"])
- # userPassword cannot be read, despite the dsHeuristic setting
+ # userPassword cannot be read, it wasn't set, instead the
+ # password was
self.assertTrue(len(res) == 1)
self.assertFalse("userPassword" in res[0])
@@ -993,7 +994,15 @@ userPassword: thatsAcomplPASS4
ldb2 = SamDB(url=host, session_info=system_session(lp),
credentials=creds, lp=lp)
- # Set userPassword to be unreadable
+ res = ldb2.search("cn=testuser,cn=users," + self.base_dn,
+ scope=SCOPE_BASE, attrs=["userPassword"])
+
+ # Check on the new connection that userPassword was not stored
+ # from ldb1 or is not readable
+ self.assertTrue(len(res) == 1)
+ self.assertFalse("userPassword" in res[0])
+
+ # Set userPassword to be readable
# This setting does not affect this connection
ldb2.set_dsheuristics("000000000")
time.sleep(1)
@@ -1014,11 +1023,10 @@ userPassword: thatsAcomplPASS4
res = ldb2.search("cn=testuser,cn=users," + self.base_dn,
scope=SCOPE_BASE, attrs=["userPassword"])
- # userPassword can be read in this connection
- # This is regardless of the current dsHeuristics setting
+ # Check despite setting it with userPassword support disabled
+ # on this connection it should still not be readable
self.assertTrue(len(res) == 1)
- self.assertTrue("userPassword" in res[0])
- self.assertEquals(res[0]["userPassword"][0], "thatsAcomplPASS2")
+ self.assertFalse("userPassword" in res[0])
# Only password from ldb1 is the user's password
creds2 = Credentials()
@@ -1050,6 +1058,17 @@ userPassword: thatsAcomplPASS4
# Reset the test "dSHeuristics" (reactivate "userPassword" pwd changes)
self.ldb.set_dsheuristics("000000001")
+ ldb4 = SamDB(url=host, session_info=system_session(lp),
+ credentials=creds, lp=lp)
+
+ # Check that userPassword that was stored from ldb2
+ res = ldb4.search("cn=testuser,cn=users," + self.base_dn,
+ scope=SCOPE_BASE, attrs=["userPassword"])
+
+ # userPassword can be not be read
+ self.assertTrue(len(res) == 1)
+ self.assertFalse("userPassword" in res[0])
+
def test_zero_length(self):
# Get the old "minPwdLength"
minPwdLength = self.ldb.get_minPwdLength()
--
2.14.3
From 91e84f5f018e58b56b13bbb268f4d93ec102a8f0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:42:17 +1300
Subject: [PATCH 03/42] ldb_wrap: Remove ldb_transaction_cancel_noerr from
ldb_wrap_fork_hook()
Writing to a TDB, without locks (these are per-process) in a forked child is never going to
end well, if a transaction is open at this point we have bigger problems.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 15 +++------------
1 file changed, 3 insertions(+), 12 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 9959b04ed95..8c3bf6f7bf3 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -329,20 +329,11 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
}
/*
- when we fork() we need to make sure that any open ldb contexts have
- any open transactions cancelled (ntdb databases doesn't need reopening,
- as we don't use clear_if_first).
- */
+ call tdb_reopen_all() in case there is a TDB open so we are
+ not blocked from re-opening it inside ldb_tdb.
+*/
void ldb_wrap_fork_hook(void)
{
- struct ldb_wrap *w;
-
- for (w=ldb_wrap_list; w; w=w->next) {
- if (ldb_transaction_cancel_noerr(w->ldb) != LDB_SUCCESS) {
- smb_panic("Failed to cancel child transactions\n");
- }
- }
-
if (tdb_reopen_all(1) != 0) {
smb_panic("tdb_reopen_all failed\n");
}
--
2.14.3
From 4f0ff1f2e7e507df0e67d4c8af22eed058fb37d9 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 9 Apr 2018 14:52:47 +1200
Subject: [PATCH 04/42] dsdb: Ensure to cancel the transaction if we fail to
save the prefixMap
This rare error case forgot to call ldb_transaction_cancel()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/repl/replicated_objects.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c
index 0c44960cf5f..4c8890f0553 100644
--- a/source4/dsdb/repl/replicated_objects.c
+++ b/source4/dsdb/repl/replicated_objects.c
@@ -921,6 +921,7 @@ WERROR dsdb_replicated_objects_commit(struct ldb_context *ldb,
}
DEBUG(0,("Failed to save updated prefixMap: %s\n",
win_errstr(werr)));
+ ldb_transaction_cancel(ldb);
TALLOC_FREE(tmp_ctx);
return werr;
}
--
2.14.3
From 523fa3fe5ad244e5ac130ecf4686c39d319b53f1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 9 Apr 2018 17:51:57 +1200
Subject: [PATCH 05/42] dsdb: Do not create a transaction in partition_init()
This will allow us to lock the databases for read during all of the Samba init
hooks.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
.../dsdb/samdb/ldb_modules/partition_metadata.c | 59 ++++++++++------------
1 file changed, 27 insertions(+), 32 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index e3ad0d8c6c2..3e60393a2c9 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -319,38 +319,6 @@ int partition_metadata_init(struct ldb_module *module)
return ret;
}
- /*
- * We need to fill in the sequence number from the DB, so we
- * need to get a lock over all the databases. We only read
- * from the main partitions, but write to metadata so to avoid
- * lock ordering we just get a transaction over the lot.
- */
- ret = partition_start_trans(module);
- if (ret != LDB_SUCCESS) {
- TALLOC_FREE(data->metadata);
- return ret;
- }
-
- ret = partition_metadata_set_sequence_number(module);
- if (ret != LDB_SUCCESS) {
- TALLOC_FREE(data->metadata);
- partition_del_trans(module);
- return ret;
- }
-
- ret = partition_prepare_commit(module);
- if (ret != LDB_SUCCESS) {
- TALLOC_FREE(data->metadata);
- partition_del_trans(module);
- return ret;
- }
-
- ret = partition_end_trans(module);
- if (ret != LDB_SUCCESS) {
- /* Nothing much we can do */
- TALLOC_FREE(data->metadata);
- }
-
return ret;
}
@@ -370,6 +338,13 @@ int partition_metadata_sequence_number(struct ldb_module *module, uint64_t *valu
return ret;
}
+ /*
+ * This means we will give a 0 until the first write
+ * tranaction, which is actually pretty reasonable.
+ *
+ * All modern databases will have the metadata.tdb from
+ * the time of the first transaction in provision anyway.
+ */
ret = partition_metadata_get_uint64(module,
LDB_METADATA_SEQ_NUM,
value,
@@ -410,6 +385,26 @@ int partition_metadata_sequence_number_increment(struct ldb_module *module, uint
return ret;
}
+ if (*value == 0) {
+ /*
+ * We are in a transaction now, so we can get the
+ * sequence number from the partitions.
+ */
+ ret = partition_metadata_set_sequence_number(module);
+ if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(data->metadata);
+ partition_del_trans(module);
+ return ret;
+ }
+
+ ret = partition_metadata_get_uint64(module,
+ LDB_METADATA_SEQ_NUM,
+ value, 0);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+ }
+
(*value)++;
ret = partition_metadata_set_uint64(module, LDB_METADATA_SEQ_NUM, *value, false);
return ret;
--
2.14.3
From 55fc4a33ceffb0ab2e67556ec302ca6e9d5db8fc Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 9 Apr 2018 21:15:25 +1200
Subject: [PATCH 06/42] dsdb: Allow search before init() call in
encrypted_secrets
Simply do not decrypt anything until the init call is run.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/encrypted_secrets.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/dsdb/samdb/ldb_modules/encrypted_secrets.c b/source4/dsdb/samdb/ldb_modules/encrypted_secrets.c
index 87ec9e4eb53..ef69bb0831c 100644
--- a/source4/dsdb/samdb/ldb_modules/encrypted_secrets.c
+++ b/source4/dsdb/samdb/ldb_modules/encrypted_secrets.c
@@ -1365,7 +1365,7 @@ static int es_search_post_process(struct ldb_module *module,
/*
* Decrypt any encrypted secret attributes
*/
- if (data->encrypt_secrets) {
+ if (data && data->encrypt_secrets) {
int err = decrypt_secret_attributes(ldb, msg, data);
if (err != LDB_SUCCESS) {
return err;
--
2.14.3
From 3b48509120a7138d878d2cb503ef4be78dcfec52 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 9 Apr 2018 21:59:01 +1200
Subject: [PATCH 07/42] dsdb: Wait until a transaction starts to call
dsdb_schema_set_indices_and_attributes()
This avoids starting a transaction in schema_load_init() and allows it
to operate with a read lock held, which will avoid locking issues
(deadlock detected due to lock odering if we do not have a global
read lock).
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
selftest/knownfail.d/dsdb_schema_attributes | 2 +
source4/dsdb/samdb/ldb_modules/schema_load.c | 60 +++++++---------------------
2 files changed, 17 insertions(+), 45 deletions(-)
create mode 100644 selftest/knownfail.d/dsdb_schema_attributes
diff --git a/selftest/knownfail.d/dsdb_schema_attributes b/selftest/knownfail.d/dsdb_schema_attributes
new file mode 100644
index 00000000000..4e9e7a05c94
--- /dev/null
+++ b/selftest/knownfail.d/dsdb_schema_attributes
@@ -0,0 +1,2 @@
+^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_attributes
+^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_indexlist
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 2099fac1159..55fb63a0f30 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -42,6 +42,12 @@ struct schema_load_private_data {
uint64_t schema_seq_num_read_lock;
uint64_t schema_seq_num_cache;
int tdb_seqnum;
+
+ /*
+ * Please write out the updated schema on the next transaction
+ * start
+ */
+ bool need_write;
};
static int dsdb_schema_from_db(struct ldb_module *module,
@@ -495,7 +501,6 @@ static int schema_load_init(struct ldb_module *module)
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct schema_load_private_data *private_data;
int ret;
- bool need_write = false;
private_data = talloc_zero(module, struct schema_load_private_data);
if (private_data == NULL) {
@@ -510,50 +515,7 @@ static int schema_load_init(struct ldb_module *module)
return ret;
}
- ret = schema_load(ldb, module, &need_write);
-
- if (ret == LDB_SUCCESS && need_write) {
- TALLOC_CTX *frame = talloc_stackframe();
- struct dsdb_schema *schema = NULL;
-
- ret = ldb_transaction_start(ldb);
- if (ret != LDB_SUCCESS) {
- ldb_debug_set(ldb, LDB_DEBUG_FATAL,
- "schema_load_init: transaction start failed");
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- schema = dsdb_get_schema(ldb, frame);
- if (schema == NULL) {
- ldb_debug_set(ldb, LDB_DEBUG_FATAL,
- "schema_load_init: dsdb_get_schema failed");
- ldb_transaction_cancel(ldb);
- return LDB_ERR_OPERATIONS_ERROR;
- }
- ret = dsdb_schema_set_indices_and_attributes(ldb, schema,
- SCHEMA_WRITE);
-
- TALLOC_FREE(frame);
-
- if (ret != LDB_SUCCESS) {
- ldb_asprintf_errstring(ldb, "Failed to write new "
- "@INDEXLIST and @ATTRIBUTES "
- "records for updated schema: %s",
- ldb_errstring(ldb));
- ldb_transaction_cancel(ldb);
- return ret;
- }
-
- ret = ldb_transaction_commit(ldb);
- if (ret != LDB_SUCCESS) {
- ldb_debug_set(ldb, LDB_DEBUG_FATAL,
- "schema_load_init: transaction commit failed");
- return LDB_ERR_OPERATIONS_ERROR;
- }
- }
-
-
- return ret;
+ return schema_load(ldb, module, &private_data->need_write);
}
static int schema_search(struct ldb_module *module, struct ldb_request *req)
@@ -586,6 +548,14 @@ static int schema_load_start_transaction(struct ldb_module *module)
"schema_load_init: dsdb_get_schema failed");
return LDB_ERR_OPERATIONS_ERROR;
}
+
+ if (private_data->need_write) {
+ ret = dsdb_schema_set_indices_and_attributes(ldb,
+ schema,
+ SCHEMA_WRITE);
+ private_data->need_write = false;
+ }
+
private_data->in_transaction++;
return ret;
--
2.14.3
From d4fdbdf2a51063ceb3296b4335564e91ef81fa6d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 11 Apr 2018 12:51:49 +1200
Subject: [PATCH 08/42] selftest: Make a transaction before @INDEXLIST etc is
checked in dsdb_schema_attributes.py
This helps us remove the write to the database from the (soon to be
read locked) init code.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/tests/dsdb_schema_attributes.py | 16 ++++++++++++++++
selftest/knownfail.d/dsdb_schema_attributes | 2 --
2 files changed, 16 insertions(+), 2 deletions(-)
delete mode 100644 selftest/knownfail.d/dsdb_schema_attributes
diff --git a/python/samba/tests/dsdb_schema_attributes.py b/python/samba/tests/dsdb_schema_attributes.py
index 2bebbb569f2..c3b956f3b99 100644
--- a/python/samba/tests/dsdb_schema_attributes.py
+++ b/python/samba/tests/dsdb_schema_attributes.py
@@ -193,6 +193,14 @@ systemOnly: FALSE
samdb2 = samba.tests.connect_samdb(self.lp.samdb_url())
+ # We now only update the @ATTRIBUTES when a transaction happens
+ # rather than making a read of the DB do writes.
+ #
+ # This avoids locking issues and is more expected
+
+ samdb2.transaction_start()
+ samdb2.transaction_commit()
+
res = self.samdb.search(base="@ATTRIBUTES", scope=ldb.SCOPE_BASE,
attrs=["@TEST_EXTRA"])
self.assertEquals(len(res), 1)
@@ -220,6 +228,14 @@ systemOnly: FALSE
samdb2 = samba.tests.connect_samdb(self.lp.samdb_url())
+ # We now only update the @INDEXLIST when a transaction happens
+ # rather than making a read of the DB do writes.
+ #
+ # This avoids locking issues and is more expected
+
+ samdb2.transaction_start()
+ samdb2.transaction_commit()
+
res = self.samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE,
attrs=["@TEST_EXTRA"])
self.assertEquals(len(res), 1)
diff --git a/selftest/knownfail.d/dsdb_schema_attributes b/selftest/knownfail.d/dsdb_schema_attributes
deleted file mode 100644
index 4e9e7a05c94..00000000000
--- a/selftest/knownfail.d/dsdb_schema_attributes
+++ /dev/null
@@ -1,2 +0,0 @@
-^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_attributes
-^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_indexlist
--
2.14.3
From 8348c9916f0bc76900cc29820419d8ab99934a57 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 10 Apr 2018 07:58:07 +1200
Subject: [PATCH 09/42] dsdb: Create rootdse_get_private_data()
This will get the private data on the first call, allowing that not to be
the init() hook.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/rootdse.c | 63 ++++++++++++++++++++++++--------
1 file changed, 48 insertions(+), 15 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/rootdse.c b/source4/dsdb/samdb/ldb_modules/rootdse.c
index 0d486218c0b..0d4cbc60f24 100644
--- a/source4/dsdb/samdb/ldb_modules/rootdse.c
+++ b/source4/dsdb/samdb/ldb_modules/rootdse.c
@@ -863,11 +863,46 @@ static int rootdse_search(struct ldb_module *module, struct ldb_request *req)
return ldb_next_request(module, down_req);
}
+static struct rootdse_private_data *rootdse_get_private_data(struct ldb_module *module)
+{
+ void *priv = ldb_module_get_private(module);
+ struct rootdse_private_data *data = NULL;
+
+ if (priv != NULL) {
+ data = talloc_get_type_abort(priv,
+ struct rootdse_private_data);
+ }
+
+ if (data != NULL) {
+ return data;
+ }
+
+ data = talloc_zero(module, struct rootdse_private_data);
+ if (data == NULL) {
+ return NULL;
+ }
+
+ data->num_controls = 0;
+ data->controls = NULL;
+ data->num_partitions = 0;
+ data->partitions = NULL;
+ data->block_anonymous = true;
+
+ ldb_module_set_private(module, data);
+ return data;
+}
+
+
static int rootdse_register_control(struct ldb_module *module, struct ldb_request *req)
{
- struct rootdse_private_data *priv = talloc_get_type(ldb_module_get_private(module), struct rootdse_private_data);
+ struct rootdse_private_data *priv =
+ rootdse_get_private_data(module);
char **list;
+ if (priv == NULL) {
+ return ldb_module_oom(module);
+ }
+
list = talloc_realloc(priv, priv->controls, char *, priv->num_controls + 1);
if (!list) {
return ldb_oom(ldb_module_get_ctx(module));
@@ -886,9 +921,14 @@ static int rootdse_register_control(struct ldb_module *module, struct ldb_reques
static int rootdse_register_partition(struct ldb_module *module, struct ldb_request *req)
{
- struct rootdse_private_data *priv = talloc_get_type(ldb_module_get_private(module), struct rootdse_private_data);
+ struct rootdse_private_data *priv =
+ rootdse_get_private_data(module);
struct ldb_dn **list;
+ if (priv == NULL) {
+ return ldb_module_oom(module);
+ }
+
list = talloc_realloc(priv, priv->partitions, struct ldb_dn *, priv->num_partitions + 1);
if (!list) {
return ldb_oom(ldb_module_get_ctx(module));
@@ -924,28 +964,21 @@ static int rootdse_request(struct ldb_module *module, struct ldb_request *req)
static int rootdse_init(struct ldb_module *module)
{
int ret;
- struct ldb_context *ldb;
struct ldb_result *res;
- struct rootdse_private_data *data;
const char *attrs[] = { "msDS-Behavior-Version", NULL };
const char *ds_attrs[] = { "dsServiceName", NULL };
TALLOC_CTX *mem_ctx;
- ldb = ldb_module_get_ctx(module);
+ struct ldb_context *ldb
+ = ldb_module_get_ctx(module);
+
+ struct rootdse_private_data *data
+ = rootdse_get_private_data(module);
- data = talloc_zero(module, struct rootdse_private_data);
if (data == NULL) {
- return ldb_oom(ldb);
+ return ldb_module_oom(module);
}
- data->num_controls = 0;
- data->controls = NULL;
- data->num_partitions = 0;
- data->partitions = NULL;
- data->block_anonymous = true;
-
- ldb_module_set_private(module, data);
-
ldb_set_default_dns(ldb);
ret = ldb_next_init(module);
--
2.14.3
From 52278657e8b97fb5560ab9132ef043fb1356537e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 10 Apr 2018 07:54:20 +1200
Subject: [PATCH 10/42] dsdb: Move ldb_set_default_dns() into
rootdse_get_private_data()
This call needs to be done at the very first chance, in this case
during the first call to the lock_read() hook, otherwise the
schema_data module can't find the schema.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/rootdse.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/rootdse.c b/source4/dsdb/samdb/ldb_modules/rootdse.c
index 0d4cbc60f24..751fe15d1a1 100644
--- a/source4/dsdb/samdb/ldb_modules/rootdse.c
+++ b/source4/dsdb/samdb/ldb_modules/rootdse.c
@@ -867,6 +867,8 @@ static struct rootdse_private_data *rootdse_get_private_data(struct ldb_module *
{
void *priv = ldb_module_get_private(module);
struct rootdse_private_data *data = NULL;
+ struct ldb_context *ldb
+ = ldb_module_get_ctx(module);
if (priv != NULL) {
data = talloc_get_type_abort(priv,
@@ -889,6 +891,9 @@ static struct rootdse_private_data *rootdse_get_private_data(struct ldb_module *
data->block_anonymous = true;
ldb_module_set_private(module, data);
+
+ ldb_set_default_dns(ldb);
+
return data;
}
@@ -979,8 +984,6 @@ static int rootdse_init(struct ldb_module *module)
return ldb_module_oom(module);
}
- ldb_set_default_dns(ldb);
-
ret = ldb_next_init(module);
if (ret != LDB_SUCCESS) {
--
2.14.3
From a6729c4b616dc6376c6a417d321f19d06f7e50cd Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 10 Apr 2018 16:34:21 +1200
Subject: [PATCH 11/42] dsdb: Allow search before init() is called in
extended_dn_out
This matches the earlier check of p && p->normalise.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/extended_dn_out.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/dsdb/samdb/ldb_modules/extended_dn_out.c b/source4/dsdb/samdb/ldb_modules/extended_dn_out.c
index ad4603fbbb9..6a869d0c482 100644
--- a/source4/dsdb/samdb/ldb_modules/extended_dn_out.c
+++ b/source4/dsdb/samdb/ldb_modules/extended_dn_out.c
@@ -498,7 +498,7 @@ static int extended_callback(struct ldb_request *req, struct ldb_reply *ares,
continue;
}
- if (p->normalise) {
+ if (p && p->normalise) {
/* If we are also in 'normalise' mode, then
* fix the attribute names to be in the
* correct case */
--
2.14.3
From b0ba46c1e4bd1519e181af4b08024bf1fbdc3567 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 11 Apr 2018 11:58:22 +1200
Subject: [PATCH 12/42] dsdb: Load schema during the read_lock() hook, not the
search
This should trigger slightly less often and is the more correct place, as
we only load it during the first lock when not in a transaction.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/schema_load.c | 19 ++++++-------------
1 file changed, 6 insertions(+), 13 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 55fb63a0f30..9ed363a5973 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -518,16 +518,6 @@ static int schema_load_init(struct ldb_module *module)
return schema_load(ldb, module, &private_data->need_write);
}
-static int schema_search(struct ldb_module *module, struct ldb_request *req)
-{
- struct ldb_context *ldb = ldb_module_get_ctx(module);
-
- /* Try the schema refresh now */
- dsdb_get_schema(ldb, NULL);
-
- return ldb_next_request(module, req);
-}
-
static int schema_load_start_transaction(struct ldb_module *module)
{
struct schema_load_private_data *private_data =
@@ -632,8 +622,10 @@ static int schema_read_lock(struct ldb_module *module)
return ldb_next_read_lock(module);
}
+ private_data->in_read_transaction++;
+
if (private_data->in_transaction == 0 &&
- private_data->in_read_transaction == 0) {
+ private_data->in_read_transaction == 1) {
/*
* This appears to fail during the init path, so do not bother
* checking the return, and return 0 (reload schema).
@@ -643,8 +635,10 @@ static int schema_read_lock(struct ldb_module *module)
&schema_seq_num, 0);
private_data->schema_seq_num_read_lock = schema_seq_num;
+
+ /* Try the schema refresh now */
+ dsdb_get_schema(ldb_module_get_ctx(module), NULL);
}
- private_data->in_read_transaction++;
return ldb_next_read_lock(module);
@@ -673,7 +667,6 @@ static const struct ldb_module_ops ldb_schema_load_module_ops = {
.name = "schema_load",
.init_context = schema_load_init,
.extended = schema_load_extended,
- .search = schema_search,
.start_transaction = schema_load_start_transaction,
.end_transaction = schema_load_end_transaction,
.del_transaction = schema_load_del_transaction,
--
2.14.3
From 9a8aacc19a3c0534327d46f7e26f65ee7979fd8a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 11 Apr 2018 12:29:18 +1200
Subject: [PATCH 13/42] dsdb: Rework schema reload during the read lock
Rather than refusing the reload based on making cached sequence numbers match
just load it once at the time the DB is globally locked, if required.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/schema_load.c | 69 ++++++++++++----------------
1 file changed, 29 insertions(+), 40 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 9ed363a5973..cfd88fec467 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -39,7 +39,6 @@ struct schema_load_private_data {
uint64_t in_transaction;
uint64_t in_read_transaction;
struct tdb_wrap *metadata;
- uint64_t schema_seq_num_read_lock;
uint64_t schema_seq_num_cache;
int tdb_seqnum;
@@ -201,16 +200,26 @@ static struct dsdb_schema *dsdb_schema_refresh(struct ldb_module *module, struct
return schema;
}
- if (private_data->in_transaction > 0) {
-
+ if (schema != NULL) {
/*
- * If the refresh is not an expected part of a larger
- * transaction, then we don't allow a schema reload during a
- * transaction. This stops others from modifying our schema
- * behind our backs
+ * If we have a schema already (not in the startup)
+ * and we are in a read or write transaction, then
+ * avoid a schema reload, it can't have changed
*/
- if (ldb_get_opaque(ldb, "dsdb_schema_refresh_expected") != (void *)1) {
- return schema;
+ if (private_data->in_transaction > 0
+ || private_data->in_read_transaction > 0 ) {
+ /*
+ * If the refresh is not an expected part of a
+ * larger transaction, then we don't allow a
+ * schema reload during a transaction. This
+ * stops others from modifying our schema
+ * behind our backs
+ */
+ if (ldb_get_opaque(ldb,
+ "dsdb_schema_refresh_expected")
+ != (void *)1) {
+ return schema;
+ }
}
}
@@ -229,19 +238,9 @@ static struct dsdb_schema *dsdb_schema_refresh(struct ldb_module *module, struct
* continue to hit the database to get the highest USN.
*/
- if (private_data->in_read_transaction > 0) {
- /*
- * We must give a static value of the metadata sequence number
- * during a read lock, otherwise, we will fail to load the
- * schema at runtime.
- */
- schema_seq_num = private_data->schema_seq_num_read_lock;
- ret = LDB_SUCCESS;
- } else {
- ret = schema_metadata_get_uint64(private_data,
- DSDB_METADATA_SCHEMA_SEQ_NUM,
- &schema_seq_num, 0);
- }
+ ret = schema_metadata_get_uint64(private_data,
+ DSDB_METADATA_SCHEMA_SEQ_NUM,
+ &schema_seq_num, 0);
if (schema != NULL) {
if (ret == LDB_SUCCESS) {
@@ -616,32 +615,26 @@ static int schema_read_lock(struct ldb_module *module)
{
struct schema_load_private_data *private_data =
talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data);
- uint64_t schema_seq_num = 0;
+ int ret;
if (private_data == NULL) {
return ldb_next_read_lock(module);
}
- private_data->in_read_transaction++;
+ ret = ldb_next_read_lock(module);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
if (private_data->in_transaction == 0 &&
- private_data->in_read_transaction == 1) {
- /*
- * This appears to fail during the init path, so do not bother
- * checking the return, and return 0 (reload schema).
- */
- schema_metadata_get_uint64(private_data,
- DSDB_METADATA_SCHEMA_SEQ_NUM,
- &schema_seq_num, 0);
-
- private_data->schema_seq_num_read_lock = schema_seq_num;
-
+ private_data->in_read_transaction == 0) {
/* Try the schema refresh now */
dsdb_get_schema(ldb_module_get_ctx(module), NULL);
}
- return ldb_next_read_lock(module);
+ private_data->in_read_transaction++;
+ return LDB_SUCCESS;
}
static int schema_read_unlock(struct ldb_module *module)
@@ -653,10 +646,6 @@ static int schema_read_unlock(struct ldb_module *module)
return ldb_next_read_unlock(module);
}
- if (private_data->in_transaction == 0 &&
- private_data->in_read_transaction == 1) {
- private_data->schema_seq_num_read_lock = 0;
- }
private_data->in_read_transaction--;
return ldb_next_read_unlock(module);
--
2.14.3
From 6530ba67f293ee1677c7576b5af51230a17e0cfb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 9 Apr 2018 18:13:59 +1200
Subject: [PATCH 14/42] dsdb: ensure we take out a read lock during the
dsdb_init
We have to also take it out in the partitions code when we load the
partition backends.
This ensures that the init handlers hold a whole-db lock just as the
search code does.
To ensure the locking count in schema_load is balanced, the
private data is now created in the first lock_read() call.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/partition.c | 24 +++++++++++++++++++++
source4/dsdb/samdb/ldb_modules/partition_init.c | 15 +++----------
source4/dsdb/samdb/ldb_modules/samba_dsdb.c | 17 +++++++++++++--
source4/dsdb/samdb/ldb_modules/schema_load.c | 28 ++++++++++++-------------
4 files changed, 55 insertions(+), 29 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c
index 37e714d6e1b..9fb1b9dee12 100644
--- a/source4/dsdb/samdb/ldb_modules/partition.c
+++ b/source4/dsdb/samdb/ldb_modules/partition.c
@@ -1236,6 +1236,30 @@ int partition_read_lock(struct ldb_module *module)
* ordering
*/
+ if (data == NULL) {
+ TALLOC_CTX *mem_ctx = talloc_new(module);
+
+ data = talloc_zero(mem_ctx, struct partition_private_data);
+ if (data == NULL) {
+ talloc_free(mem_ctx);
+ return ldb_operr(ldb);
+ }
+
+ /*
+ * When used from Samba4, this message is set by the
+ * samba4 module, as a fixed value not read from the
+ * DB. This avoids listing modules in the DB
+ */
+ data->forced_module_msg = talloc_get_type(
+ ldb_get_opaque(ldb,
+ DSDB_OPAQUE_PARTITION_MODULE_MSG_OPAQUE_NAME),
+ struct ldb_message);
+
+ ldb_module_set_private(module, talloc_steal(module,
+ data));
+ talloc_free(mem_ctx);
+ }
+
/*
* This will lock the metadata partition (sam.ldb) and
* will also call event loops, so we do it before we
diff --git a/source4/dsdb/samdb/ldb_modules/partition_init.c b/source4/dsdb/samdb/ldb_modules/partition_init.c
index 9a6ac0c05a9..9a8bb7e211d 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_init.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_init.c
@@ -863,18 +863,9 @@ int partition_init(struct ldb_module *module)
return ldb_operr(ldb);
}
- data = talloc_zero(mem_ctx, struct partition_private_data);
- if (data == NULL) {
- return ldb_operr(ldb);
- }
-
- /* When used from Samba4, this message is set by the samba4
- * module, as a fixed value not read from the DB. This avoids
- * listing modules in the DB */
- data->forced_module_msg = talloc_get_type(
- ldb_get_opaque(ldb,
- DSDB_OPAQUE_PARTITION_MODULE_MSG_OPAQUE_NAME),
- struct ldb_message);
+ /* We actually got this during the read_lock call */
+ data = talloc_get_type_abort(ldb_module_get_private(module),
+ struct partition_private_data);
/* This loads the partitions */
ret = partition_reload_if_required(module, data, NULL);
diff --git a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
index 2605c1e511e..54ec6a26068 100644
--- a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
+++ b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
@@ -249,7 +249,7 @@ static bool check_required_features(struct ldb_message_element *el)
static int samba_dsdb_init(struct ldb_module *module)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
- int ret, len, i, j;
+ int ret, lock_ret, len, i, j;
TALLOC_CTX *tmp_ctx = talloc_new(module);
struct ldb_result *res;
struct ldb_message *rootdse_msg = NULL, *partition_msg;
@@ -627,7 +627,20 @@ static int samba_dsdb_init(struct ldb_module *module)
/* Set this as the 'next' module, so that we effectivly append it to module chain */
ldb_module_set_next(module, module_chain);
- return ldb_next_init(module);
+ ret = ldb_next_read_lock(module);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
+ ret = ldb_next_init(module);
+
+ lock_ret = ldb_next_read_unlock(module);
+
+ if (lock_ret != LDB_SUCCESS) {
+ return lock_ret;
+ }
+
+ return ret;
}
static const struct ldb_module_ops ldb_samba_dsdb_module_ops = {
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index cfd88fec467..e977e8b0ad5 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -498,17 +498,11 @@ static int schema_load(struct ldb_context *ldb,
static int schema_load_init(struct ldb_module *module)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
- struct schema_load_private_data *private_data;
+ struct schema_load_private_data *private_data =
+ talloc_get_type_abort(ldb_module_get_private(module),
+ struct schema_load_private_data);
int ret;
- private_data = talloc_zero(module, struct schema_load_private_data);
- if (private_data == NULL) {
- return ldb_oom(ldb);
- }
- private_data->module = module;
-
- ldb_module_set_private(module, private_data);
-
ret = ldb_next_init(module);
if (ret != LDB_SUCCESS) {
return ret;
@@ -618,7 +612,14 @@ static int schema_read_lock(struct ldb_module *module)
int ret;
if (private_data == NULL) {
- return ldb_next_read_lock(module);
+ private_data = talloc_zero(module, struct schema_load_private_data);
+ if (private_data == NULL) {
+ return ldb_module_oom(module);
+ }
+
+ private_data->module = module;
+
+ ldb_module_set_private(module, private_data);
}
ret = ldb_next_read_lock(module);
@@ -640,11 +641,8 @@ static int schema_read_lock(struct ldb_module *module)
static int schema_read_unlock(struct ldb_module *module)
{
struct schema_load_private_data *private_data =
- talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data);
-
- if (private_data == NULL) {
- return ldb_next_read_unlock(module);
- }
+ talloc_get_type_abort(ldb_module_get_private(module),
+ struct schema_load_private_data);
private_data->in_read_transaction--;
--
2.14.3
From 689a464e0f24c92947d4ba94ba21719c16faf322 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 10 Apr 2018 13:34:56 +1200
Subject: [PATCH 15/42] dsdb: Use talloc_get_type_abort() in
schema_load_{start,end}_transaction
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13379
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/schema_load.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index e977e8b0ad5..7cbbeb699ee 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -514,7 +514,8 @@ static int schema_load_init(struct ldb_module *module)
static int schema_load_start_transaction(struct ldb_module *module)
{
struct schema_load_private_data *private_data =
- talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data);
+ talloc_get_type_abort(ldb_module_get_private(module),
+ struct schema_load_private_data);
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct dsdb_schema *schema;
int ret;
@@ -547,7 +548,8 @@ static int schema_load_start_transaction(struct ldb_module *module)
static int schema_load_end_transaction(struct ldb_module *module)
{
struct schema_load_private_data *private_data =
- talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data);
+ talloc_get_type_abort(ldb_module_get_private(module),
+ struct schema_load_private_data);
struct ldb_context *ldb = ldb_module_get_ctx(module);
if (private_data->in_transaction == 0) {
--
2.14.3
From d992f59d82766871200d671f2fd47252b95ba6fa Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:44:52 +1300
Subject: [PATCH 16/42] ldb_wrap: Remove the magic cache of database handles
except for sam.ldb
sam.ldb is handled in samdb_connect_url(), not this function.
This cache caused issues when "private dir" was changed in a testing script, but also
just generates many-owner shared mutable state that is frowned upon these days.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 8c3bf6f7bf3..53255556e7c 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -303,9 +303,13 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
struct ldb_context *ldb;
int ret;
- ldb = ldb_wrap_find(url, ev, lp_ctx, session_info, credentials, flags);
- if (ldb != NULL)
- return talloc_reference(mem_ctx, ldb);
+ /*
+ * Unlike samdb_connect_url() do not try and cache the LDB
+ * handle, get a new one each time. Only sam.ldb is
+ * punitively expensive to open and helpful caches like this
+ * cause challenges (such as if the value for 'private dir'
+ * changes).
+ */
ldb = samba_ldb_init(mem_ctx, ev, lp_ctx, session_info, credentials);
@@ -318,11 +322,6 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
return NULL;
}
- if (!ldb_wrap_add(url, ev, lp_ctx, session_info, credentials, flags, ldb)) {
- talloc_free(ldb);
- return NULL;
- }
-
DEBUG(3,("ldb_wrap open of %s\n", url));
return ldb;
--
2.14.3
From cb607346d3c7c662343b0eae69e43eaa6358c188 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 16:43:54 +1300
Subject: [PATCH 17/42] ldb-samba: require pid match for cached ldb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 53255556e7c..143e128d5d7 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -94,6 +94,8 @@ static struct ldb_wrap {
/* the context is what we use to tell if two ldb
* connections are exactly equivalent
*/
+ pid_t pid; /* We want to re-open in a new PID due to
+ * the LMDB backend */
const char *url;
struct tevent_context *ev;
struct loadparm_context *lp_ctx;
@@ -186,10 +188,12 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
struct cli_credentials *credentials,
unsigned int flags)
{
+ pid_t pid = getpid();
struct ldb_wrap *w;
/* see if we can re-use an existing ldb */
for (w=ldb_wrap_list; w; w=w->next) {
- if (w->context.ev == ev &&
+ if (w->context.pid == pid &&
+ w->context.ev == ev &&
w->context.lp_ctx == lp_ctx &&
w->context.session_info == session_info &&
w->context.credentials == credentials &&
@@ -249,6 +253,7 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
return false;
}
+ c.pid = getpid();
c.url = url;
c.ev = ev;
c.lp_ctx = lp_ctx;
--
2.14.3
From 0e932a5ce45bbf1461ce30a2f8eff7d573cb757d Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 18/42] ldb_mdb: Implement the lmdb backend for ldb
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 711 ++++++++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 57 ++++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
lib/ldb/tools/ldbdump.c | 126 +++++++-
lib/ldb/wscript | 92 +++++-
5 files changed, 978 insertions(+), 9 deletions(-)
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 00000000000..8bc82715390
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,711 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "lmdb.h"
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX "mdb://"
+#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+
+#define MEGABYTE (1024*1024)
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+ switch (lmdb_err) {
+ case MDB_SUCCESS:
+ return LDB_SUCCESS;
+ case EIO:
+ return LDB_ERR_OPERATIONS_ERROR;
+ case MDB_INCOMPATIBLE:
+ case MDB_CORRUPTED:
+ case MDB_INVALID:
+ return LDB_ERR_UNAVAILABLE;
+ case MDB_BAD_TXN:
+ case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+ case MDB_BAD_DBI:
+#endif
+ case MDB_PANIC:
+ case EINVAL:
+ return LDB_ERR_PROTOCOL_ERROR;
+ case MDB_MAP_FULL:
+ case MDB_DBS_FULL:
+ case MDB_READERS_FULL:
+ case MDB_TLS_FULL:
+ case MDB_TXN_FULL:
+ case EAGAIN:
+ return LDB_ERR_BUSY;
+ case MDB_KEYEXIST:
+ return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ case MDB_NOTFOUND:
+ case ENOENT:
+ return LDB_ERR_NO_SUCH_OBJECT;
+ case EACCES:
+ return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+ default:
+ break;
+ }
+ return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+ int ecode,
+ const char *file,
+ int line)
+{
+ int ldb_err = ldb_mdb_err_map(ecode);
+ char *reason = mdb_strerror(ecode);
+ ldb_asprintf_errstring(ldb,
+ "(%d) - %s at %s:%d",
+ ecode,
+ reason,
+ file,
+ line);
+ return ldb_err;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+ if (ltx == NULL) {
+ return NULL;
+ }
+
+ return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ if (lmdb->txlist) {
+ talloc_steal(lmdb->txlist, ltx);
+ }
+
+ DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ DLIST_REMOVE(lmdb->txlist, ltx);
+ talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx;
+
+ ltx = lmdb->txlist;
+ return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+ MDB_txn *txn;
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
+ }
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ int ret;
+ ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ lmdb->error = ret;
+ ldb_asprintf_errstring(lmdb->ldb,
+ "%s failed: %s\n", __FUNCTION__,
+ mdb_strerror(ret));
+ }
+ lmdb->read_txn = txn;
+ }
+ return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val data, int flags)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ int mdb_flags;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ mdb_data.mv_size = data.length;
+ mdb_data.mv_data = data.data;
+
+ if (flags == TDB_INSERT) {
+ mdb_flags = MDB_NOOVERWRITE;
+ } else if ((flags == TDB_MODIFY)) {
+ /*
+ * Modifying a record, ensure that it exists.
+ * This mimics the TDB semantics
+ */
+ MDB_val value;
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+ if (lmdb->error != MDB_SUCCESS) {
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ mdb_flags = 0;
+ } else {
+ mdb_flags = 0;
+ }
+
+ lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+ ldb_kv_traverse_fn fn,
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+ MDB_cursor *cursor = NULL;
+ int ret;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+ if (lmdb->error != MDB_SUCCESS) {
+ goto done;
+ }
+
+ while ((lmdb->error = mdb_cursor_get(
+ cursor, &mdb_key,
+ &mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+ struct ldb_val key = {
+ .length = mdb_key.mv_size,
+ .data = mdb_key.mv_data,
+ };
+ struct ldb_val data = {
+ .length = mdb_data.mv_size,
+ .data = mdb_data.mv_data,
+ };
+
+ ret = fn(ltdb, key, data, ctx);
+ if (ret != 0) {
+ goto done;
+ }
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ lmdb->error = MDB_SUCCESS;
+ }
+done:
+ if (cursor != NULL) {
+ mdb_cursor_close(cursor);
+ }
+
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val key2,
+ struct ldb_val data,
+ void *state)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct ldb_val copy;
+ int ret = LDB_SUCCESS;
+
+ /*
+ * Need to take a copy of the data as the delete operation alters the
+ * data, as it is in private lmdb memory.
+ */
+ copy.length = data.length;
+ copy.data = talloc_memdup(ltdb, data.data, data.length);
+ if (copy.data == NULL) {
+ lmdb->error = MDB_PANIC;
+ return ldb_oom(lmdb->ldb);
+ }
+
+ lmdb->error = lmdb_delete(ltdb, key);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to delete %*.*s "
+ "for rekey as %*.*s: %s",
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+ lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to rekey %*.*s as %*.*s: %s",
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+
+done:
+ if (copy.data != NULL) {
+ TALLOC_FREE(copy.data);
+ copy.length = 0;
+ }
+
+ /*
+ * Explicity invalidate the data, as the delete has done this
+ */
+ data.length = 0;
+ data.data = NULL;
+
+ return ret;
+}
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
+ void *private_data),
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct ldb_val data;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction active");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+ if (lmdb->error != MDB_SUCCESS) {
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ data.data = mdb_data.mv_data;
+ data.length = mdb_data.mv_size;
+
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+
+ /* We created a read transaction, commit it */
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ lmdb->error = MDB_SUCCESS;
+ if (ltdb->in_transaction == 0 &&
+ ltdb->read_lock_count == 0) {
+ lmdb->error = mdb_txn_begin(lmdb->env,
+ NULL,
+ MDB_RDONLY,
+ &lmdb->read_txn);
+ }
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ ltdb->read_lock_count++;
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+ if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+ }
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct lmdb_trans *ltx;
+ struct lmdb_trans *ltx_head;
+ MDB_txn *tx_parent;
+
+ ltx = talloc_zero(lmdb, struct lmdb_trans);
+ if (ltx == NULL) {
+ return ldb_oom(lmdb->ldb);
+ }
+
+ ltx_head = lmdb_private_trans_head(lmdb);
+
+ tx_parent = lmdb_trans_get_tx(ltx_head);
+
+ lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, <x->tx);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ trans_push(lmdb, ltx);
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+ /* No need to prepare a commit */
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ lmdb->error = mdb_txn_commit(ltx->tx);
+ trans_finished(lmdb, ltx);
+
+ return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+ return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+ return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+ return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+ /*
+ * lmdb does no provide a quick way to determine if the database
+ * has changed. This function always returns true.
+ *
+ * Note that tdb uses a sequence number that allows this function
+ * to be implemented efficiently.
+ */
+ return true;
+}
+
+
+static struct kv_db_ops lmdb_key_value_ops = {
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+ const char *path;
+
+ /* parse the url */
+ if (strchr(url, ':')) {
+ if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+ return NULL;
+ }
+ path = url + MDB_URL_PREFIX_SIZE;
+ } else {
+ path = url;
+ }
+
+ return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx = NULL;
+
+ /*
+ * Close the read transaction if it's open
+ */
+ if (lmdb->read_txn != NULL) {
+ mdb_txn_abort(lmdb->read_txn);
+ }
+
+ if (lmdb->env == NULL) {
+ return 0;
+ }
+
+ /*
+ * Abort any currently active transactions
+ */
+ ltx = lmdb_private_trans_head(lmdb);
+ while (ltx != NULL) {
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ ltx = lmdb_private_trans_head(lmdb);
+ }
+
+ mdb_env_close(lmdb->env);
+ lmdb->env = NULL;
+
+ return 0;
+}
+
+static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path)
+{
+ struct lmdb_private *lmdb;
+ int ret;
+
+ lmdb = talloc_zero(ldb, struct lmdb_private);
+ if (lmdb == NULL) {
+ return NULL;
+ }
+ lmdb->ldb = ldb;
+
+ ret = mdb_env_create(&lmdb->env);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not create MDB environment %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+ ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ if (ret != 0) {
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ mdb_env_set_maxreaders(lmdb->env, 100000);
+ /* MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+ ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not open DB %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ return lmdb;
+
+}
+
+static int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module)
+{
+ const char *path = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct ltdb_private *ltdb = NULL;
+
+ /*
+ * We hold locks, so we must use a private event context
+ * on each returned handle
+ */
+ ldb_set_require_private_event_context(ldb);
+
+ path = lmdb_get_path(url);
+ if (path == NULL) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb = talloc_zero(ldb, struct ltdb_private);
+ if (!ltdb) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ lmdb = lmdb_pvt_create(ldb, ldb, path);
+ if (lmdb == NULL) {
+ ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb->lmdb_private = lmdb;
+ if (flags & LDB_FLG_RDONLY) {
+ ltdb->read_only = true;
+ }
+ return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
+
+
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 00000000000..18d28e822d7
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,57 @@
+/*
+ ldb database library using mdb back end - transaction operations
+
+ Copyright (C) Jakub Hrozek 2015
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include <lmdb.h>
+
+#include "ldb_private.h"
+
+
+struct lmdb_private {
+ struct ldb_context *ldb;
+ MDB_env *env;
+
+ struct lmdb_trans *txlist;
+
+ struct ldb_mdb_metadata {
+ struct ldb_message *attributes;
+ unsigned seqnum;
+ } *meta;
+ int error;
+ MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+ struct lmdb_trans *next;
+ struct lmdb_trans *prev;
+
+ MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f14666ba88a..7d92804a8e9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
struct ltdb_private {
const struct kv_db_ops *kv_ops;
TDB_CONTEXT *tdb;
+ struct lmdb_private *lmdb_private;
unsigned int connect_flags;
unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59eca4..2da2ca8ec70 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
#include <ldb.h>
#include <ldb_private.h>
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
static struct ldb_context *ldb;
bool show_index = false;
bool validate_contents = false;
@@ -166,6 +171,116 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
}
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ int ret;
+ struct MDB_env *env = NULL;
+ struct MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct MDB_cursor *cursor = NULL;
+ struct MDB_val key;
+ struct MDB_val data;
+
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not create MDB environment: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_env_open(env,
+ fname,
+ MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY,
+ 0600);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open environment for %s: (%d) %s\n",
+ fname,
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not start transaction: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open database: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_open(txn, dbi, &cursor);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open cursor: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not find first record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ while (ret != MDB_NOTFOUND) {
+ struct TDB_DATA tkey = {
+ .dptr = key.mv_data,
+ .dsize = key.mv_size
+ };
+ struct TDB_DATA tdata = {
+ .dptr = data.mv_data,
+ .dsize = data.mv_size
+ };
+ traverse_fn(NULL, tkey, tdata, dn);
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not read next record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ }
+ ret = 0;
+
+close_cursor:
+ mdb_cursor_close(cursor);
+close_txn:
+ mdb_txn_commit(txn);
+close_env:
+ mdb_env_close(env);
+
+ if (ret != 0) {
+ return 1;
+ }
+ return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ /* not built with lmdb support */
+ return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
static void usage( void)
{
printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +344,14 @@ static void usage( void)
fname = argv[optind];
- return dump_tdb(fname, dn, emergency);
+ rc = dump_lmdb(fname, dn, emergency);
+ if (rc != 0) {
+ rc = dump_tdb(fname, dn, emergency);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to open %s\n", fname);
+ return 1;
+ }
+ }
+ return 0;
+
}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index aab4da132d5..d7907596612 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
srcdir = srcdir + '/..'
sys.path.insert(0, srcdir + '/buildtools/wafsamba')
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -92,6 +92,16 @@ def configure(conf):
implied_deps='replace talloc tdb tevent'):
conf.define('USING_SYSTEM_LDB', 1)
+ if conf.env.standalone_ldb:
+ # Require lmdb support for standalone mode.
+ conf.env.REQUIRE_LMDB = True
+ elif not Options.options.without_ad_dc:
+ # Require lmdb support for addc mode
+ conf.env.REQUIRE_LMDB = True
+ else:
+ conf.env.REQUIRE_LMDB = False
+
+
if conf.CONFIG_SET('USING_SYSTEM_LDB'):
v = VERSION.split('.')
conf.DEFINE('EXPECTED_SYSTEM_LDB_VERSION_MAJOR', int(v[0]))
@@ -110,6 +120,36 @@ def configure(conf):
if not sys.platform.startswith("openbsd"):
conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
+ # if lmdb support is enabled then we require lmdb
+ # is present, build the mdb back end and enable lmdb support in
+ # the tools.
+ if conf.env.REQUIRE_LMDB:
+ if not conf.CHECK_CFG(package='lmdb',
+ args='"lmdb >= 0.9.16" --cflags --libs',
+ msg='Checking for lmdb >= 0.9.16',
+ mandatory=False):
+ if not conf.CHECK_CODE('''
+ #if MDB_VERSION_MAJOR == 0 \
+ && MDB_VERSION_MINOR <= 9 \
+ && MDB_VERSION_PATCH < 16
+ #error LMDB too old
+ #endif
+ ''',
+ 'HAVE_GOOD_LMDB_VERSION',
+ headers='lmdb.h',
+ msg='Checking for lmdb >= 0.9.16 via header check'):
+
+ if conf.env.standalone_ldb:
+ raise Utils.WafError('ldb requires '
+ 'lmdb 0.9.16 or later')
+ elif not Options.options.without_ad_dc:
+ raise Utils.WafError('Samba AD DC requires '
+ 'lmdb 0.9.16 or later')
+
+ if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+ conf.DEFINE('HAVE_LMDB', '1')
+ conf.env.ENABLE_MDB_BACKEND = True
+
conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
conf.SAMBA_CONFIG_H()
@@ -321,6 +361,20 @@ def build(bld):
private_library=True,
deps='tdb ldb')
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_MODULE('ldb_mdb',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ init_function='ldb_mdb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value lmdb',
+ subsystem='ldb')
+ lmdb_deps = ' ldb_mdb_int'
+ else:
+ lmdb_deps = ''
+
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
@@ -338,8 +392,14 @@ def build(bld):
bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ lmdb_deps = ' lmdb'
+ else:
+ lmdb_deps = ''
# ldbdump doesn't get installed
- bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+ bld.SAMBA_BINARY('ldbdump',
+ 'tools/ldbdump.c',
+ deps='ldb-cmdline ldb' + lmdb_deps,
install=False)
bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -365,6 +425,18 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+ source='tests/ldb_mod_op_test.c',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
+ deps='cmocka ldb',
+ install=False)
+ bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
+ source='tests/ldb_kv_ops_test.c',
+ cflags='-DTEST_BE=\"mdb\"',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
deps='cmocka ldb',
@@ -402,12 +474,16 @@ def test(ctx):
print("Python testsuite returned %d" % pyret)
cmocka_ret = 0
- for test_exe in ['test_ldb_qsort',
- 'ldb_msg_test',
- 'ldb_tdb_mod_op_test',
- 'ldb_tdb_guid_mod_op_test',
- 'ldb_msg_test',
- 'ldb_tdb_kv_ops_test']:
+ test_exes = ['test_ldb_qsort',
+ 'ldb_msg_test',
+ 'ldb_tdb_mod_op_test',
+ 'ldb_tdb_guid_mod_op_test',
+ 'ldb_tdb_kv_ops_test']
+
+ if env.ENABLE_MDB_BACKEND:
+ test_exes.append('ldb_mdb_mod_op_test')
+ test_exes.append('ldb_mdb_kv_ops_test')
+ for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.14.3
From 15ecfdfb4000bf0e6e78344b7ea8237a85f10a95 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 19/42] ldb: Handle multiple ldb url schemes
Here we use ldb_relative_path more often and allow for the stripping of
ldb:// and mdb://.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 6 ++++++
source4/dsdb/samdb/ldb_modules/partition_metadata.c | 20 +++++++++-----------
source4/dsdb/samdb/ldb_modules/schema_load.c | 19 +++++--------------
3 files changed, 20 insertions(+), 25 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 143e128d5d7..4dc5ca6da2a 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -356,6 +356,12 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
if (strncmp("tdb://", base_url, 6) == 0) {
base_url = base_url+6;
}
+ else if (strncmp("mdb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
+ else if (strncmp("ldb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
path = talloc_strdup(mem_ctx, base_url);
if (path == NULL) {
return NULL;
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index 3e60393a2c9..bc6503c6789 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -18,6 +18,7 @@
*/
#include "dsdb/samdb/ldb_modules/partition.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
#define LDB_METADATA_SEQ_NUM "SEQ_NUM"
@@ -185,7 +186,6 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
TALLOC_CTX *tmp_ctx;
struct partition_private_data *data;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename, *dirname;
int open_flags;
struct stat statbuf;
@@ -202,15 +202,11 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
+
if (!filename) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
@@ -222,7 +218,9 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
/* While provisioning, sam.ldb.d directory may not exist,
* so create it. Ignore errors, if it already exists. */
- dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name);
+ dirname = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d");
if (!dirname) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 7cbbeb699ee..2f448dbba58 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -32,6 +32,7 @@
#include <tdb.h>
#include "lib/tdb_wrap/tdb_wrap.h"
#include "dsdb/samdb/ldb_modules/util.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
struct schema_load_private_data {
@@ -63,7 +64,6 @@ static int schema_metadata_open(struct ldb_module *module)
struct ldb_context *ldb = ldb_module_get_ctx(module);
TALLOC_CTX *tmp_ctx;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename;
int open_flags;
struct stat statbuf;
@@ -79,19 +79,10 @@ static int schema_metadata_open(struct ldb_module *module)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
- if (!filename) {
- talloc_free(tmp_ctx);
- return ldb_oom(ldb);
- }
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
open_flags = O_RDWR;
if (stat(filename, &statbuf) != 0) {
--
2.14.3
From 9cd272a5385caf9ea6248f8119128c4371081775 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 20/42] ldb: Introduce new ldb:// prefix to allow failover
If you try to open with tdb first you find that you can clobber a
database due to the default tdb behaviour.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_ldb/ldb_ldb.c | 78 +++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.c | 79 +++++++++++++++++++++++++-----------------
lib/ldb/ldb_mdb/ldb_mdb.h | 3 ++
lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 +++++++++++++++++
lib/ldb/wscript | 19 ++++++++--
5 files changed, 176 insertions(+), 34 deletions(-)
create mode 100644 lib/ldb/ldb_ldb/ldb_ldb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c
diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
new file mode 100644
index 00000000000..a63967ff8d6
--- /dev/null
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -0,0 +1,78 @@
+/*
+ * ldb connection and module initialisation
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#include "ldb_private.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+
+/*
+ connect to the database
+*/
+static int lldb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **module)
+{
+ const char *path;
+ int ret;
+
+ /*
+ * Check and remove the url prefix
+ */
+ if (strchr(url, ':')) {
+ if (strncmp(url, "ldb://", 6) != 0) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR,
+ "Invalid ldb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ path = url+6;
+ } else {
+ path = url;
+ }
+
+ /*
+ * Don't create the database if it's not there
+ */
+ flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+ /*
+ * Try opening the database as an lmdb
+ */
+ ret = lmdb_connect(ldb, path, flags, options, module);
+ if (ret == LDB_SUCCESS) {
+ return ret;
+ }
+ if (ret == LDB_ERR_UNAVAILABLE) {
+ /*
+ * Not mdb so try as tdb
+ */
+ ret = ltdb_connect(ldb, path, flags, options, module);
+ }
+#else
+ ret = ltdb_connect(ldb, path, flags, options, module);
+#endif
+ return ret;
+}
+
+int ldb_ldb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("ldb", lldb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 8bc82715390..314c78ff626 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -614,26 +614,30 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
return 0;
}
-static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
- struct ldb_context *ldb,
- const char *path)
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags,
+ struct lmdb_private *lmdb)
{
- struct lmdb_private *lmdb;
int ret;
+ unsigned int mdb_flags;
- lmdb = talloc_zero(ldb, struct lmdb_private);
- if (lmdb == NULL) {
- return NULL;
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
}
- lmdb->ldb = ldb;
ret = mdb_env_create(&lmdb->env);
if (ret != 0) {
- ldb_asprintf_errstring(ldb,
- "Could not create MDB environment %s: %s\n",
- path, mdb_strerror(ret));
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
}
/* Close when lmdb is released */
@@ -641,34 +645,45 @@ static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
if (ret != 0) {
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not open MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return ldb_mdb_err_map(ret);
}
mdb_env_set_maxreaders(lmdb->env, 100000);
/* MDB_NOSUBDIR implies there is a separate file called path and a
* separate lockfile called path-lock
*/
- ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ if (flags & LDB_FLG_RDONLY) {
+ mdb_flags |= MDB_RDONLY;
+ }
+ ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
talloc_free(lmdb);
- return NULL;
+ return ldb_mdb_err_map(ret);
}
- return lmdb;
+ return LDB_SUCCESS;
}
-static int lmdb_connect(struct ldb_context *ldb, const char *url,
- unsigned int flags, const char *options[],
- struct ldb_module **_module)
+int lmdb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **_module)
{
const char *path = NULL;
struct lmdb_private *lmdb = NULL;
struct ltdb_private *ltdb = NULL;
+ int ret;
/*
* We hold locks, so we must use a private event context
@@ -687,12 +702,19 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
}
- ltdb->kv_ops = &lmdb_key_value_ops;
- lmdb = lmdb_pvt_create(ldb, ldb, path);
+ lmdb = talloc_zero(ldb, struct lmdb_private);
if (lmdb == NULL) {
- ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
- return LDB_ERR_OPERATIONS_ERROR;
+ TALLOC_FREE(ltdb);
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ lmdb->ldb = ldb;
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ if (ret != LDB_SUCCESS) {
+ return ret;
}
ltdb->lmdb_private = lmdb;
@@ -702,10 +724,3 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
-
-
-int ldb_mdb_init(const char *version)
-{
- LDB_MODULE_CHECK_VERSION(version);
- return ldb_register_backend("mdb", lmdb_connect, false);
-}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 18d28e822d7..e62e353b54e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -53,5 +53,8 @@ struct lmdb_trans {
};
int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module);
#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 00000000000..339c3f22b2a
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index d7907596612..2ef6305eba2 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -364,17 +364,32 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_MODULE('ldb_mdb',
bld.SUBDIR('ldb_mdb',
- '''ldb_mdb.c '''),
+ '''ldb_mdb_init.c'''),
init_function='ldb_mdb_init',
module_init_name='ldb_init_module',
internal_module=False,
- deps='ldb ldb_key_value lmdb',
+ deps='ldb ldb_key_value ldb_mdb_int',
subsystem='ldb')
+
+ bld.SAMBA_LIBRARY('ldb_mdb_int',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ private_library=True,
+ deps='ldb lmdb ldb_key_value')
lmdb_deps = ' ldb_mdb_int'
else:
lmdb_deps = ''
+ bld.SAMBA_MODULE('ldb_ldb',
+ bld.SUBDIR('ldb_ldb',
+ '''ldb_ldb.c'''),
+ init_function='ldb_ldb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value' + lmdb_deps,
+ subsystem='ldb')
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
--
2.14.3
From 0572a61ba0eee4795036c8ff128597a8ae6706f4 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 21/42] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb
This is used in selftest with 'ldb:nosync = true'.
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 314c78ff626..a34cf35a7d5 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -661,6 +661,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
if (flags & LDB_FLG_RDONLY) {
mdb_flags |= MDB_RDONLY;
}
+ if (flags & LDB_FLG_NOSYNC) {
+ mdb_flags |= MDB_NOSYNC;
+ }
ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
--
2.14.3
From 1af113235adbc058223c1ac4350a9e45789d1db1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 22/42] tests: Add tests to check for max key length and DB
size
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_size_test.c | 213 ++++++++++++++++++++++++++++++++
lib/ldb/tests/ldb_lmdb_test.c | 243 +++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 12 ++
3 files changed, 468 insertions(+)
create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
create mode 100644 lib/ldb/tests/ldb_lmdb_test.c
diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 00000000000..506c586289e
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,213 @@
+/*
+ * lmdb backend specific tests for ldb
+ * Tests for truncated index keys
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+ int ret, x;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ const int MB = 1024 * 1024;
+ char *blob = NULL;
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ blob = talloc_zero_size(tmp_ctx, (MB + 1));
+ assert_non_null(blob);
+ memset(blob, 'x', MB);
+
+
+ for (x = 0; x < 6144; x++) {
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+ assert_non_null(msg->dn);
+
+ ldb_transaction_start(test_ctx->ldb);
+ ret = ldb_msg_add_string(msg, "blob", blob);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+ ldb_transaction_commit(test_ctx->ldb);
+
+ TALLOC_FREE(msg);
+ }
+ talloc_free(tmp_ctx);
+ {
+ struct stat s;
+ ret = stat(test_ctx->dbfile, &s);
+ assert_int_equal(ret, 0);
+ assert_true(s.st_size > (6144LL * MB));
+ }
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_db_size_gt_4GB,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 00000000000..2902eaf0533
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,243 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_eq_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 2ef6305eba2..577203beefb 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -446,6 +446,17 @@ def build(bld):
cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
deps='cmocka ldb',
install=False)
+
+ bld.SAMBA_BINARY('ldb_lmdb_test',
+ source='tests/ldb_lmdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+
+ bld.SAMBA_BINARY('ldb_lmdb_size_test',
+ source='tests/ldb_lmdb_size_test.c',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
source='tests/ldb_kv_ops_test.c',
cflags='-DTEST_BE=\"mdb\"',
@@ -497,6 +508,7 @@ def test(ctx):
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
+ test_exes.append('ldb_lmdb_test')
test_exes.append('ldb_mdb_kv_ops_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
--
2.14.3
From 53a01482510c4491e46ba623f93a064909cded0b Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 23/42] ldb_mdb: Store pid to change destructor on fork
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 2 ++
2 files changed, 24 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index a34cf35a7d5..711700c2697 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -587,6 +587,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
{
struct lmdb_trans *ltx = NULL;
+ /* Check if this is a forked child */
+ if (getpid() != lmdb->pid) {
+ int fd = 0;
+ /*
+ * We cannot call mdb_env_close or commit any transactions,
+ * otherwise they might appear finished in the parent.
+ *
+ */
+
+ if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+ close(fd);
+ }
+
+ /* Remove the pointer, so that no access should occur */
+ lmdb->env = NULL;
+
+ return 0;
+ }
+
/*
* Close the read transaction if it's open
*/
@@ -673,6 +692,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
return ldb_mdb_err_map(ret);
}
+ /* Store the original pid during the LMDB open */
+ lmdb->pid = getpid();
+
return LDB_SUCCESS;
}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index e62e353b54e..7e0729cc7af 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -43,6 +43,8 @@ struct lmdb_private {
int error;
MDB_txn *read_txn;
+ pid_t pid;
+
};
struct lmdb_trans {
--
2.14.3
From 81776ec48d3ed36a2ab4972f8d760ac8057b6ad1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 24/42] lib ldb tests: Run api and index test on tdb and lmdb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 87 ++++++++++++++++++++++++++++++++++++++++++-
lib/ldb/tests/python/index.py | 11 ++++++
2 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 12409a8c991..9a2997f5b48 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -633,6 +633,16 @@ class SimpleLdb(LdbBaseTest):
l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SimpleLdbLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SimpleLdbLmdb, self).tearDown()
+
class SearchTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -1057,6 +1067,17 @@ class SearchTests(LdbBaseTest):
self.assertEqual(len(res11), 1)
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SearchTestsLmdb, self).tearDown()
+
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1152,6 +1173,35 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
class AddModifyTests(LdbBaseTest):
def tearDown(self):
@@ -1296,6 +1346,15 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
+class AddModifyTestsLmdb(AddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(AddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(AddModifyTestsLmdb, self).tearDown()
+
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1407,6 +1466,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
self.l.transaction_commit()
super(TransIndexedAddModifyTests, self).tearDown()
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
class BadIndexTests(LdbBaseTest):
def setUp(self):
@@ -1567,7 +1643,6 @@ class GUIDBadIndexTests(BadIndexTests):
super(GUIDBadIndexTests, self).setUp()
-
class DnTests(TestCase):
def setUp(self):
@@ -2429,6 +2504,16 @@ class LdbResultTests(LdbBaseTest):
self.assertEqual(got_pid, pid)
+class LdbResultTestsLmdb(LdbResultTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(LdbResultTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(LdbResultTestsLmdb, self).tearDown()
+
+
class BadTypeTests(TestCase):
def test_control(self):
l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 9b9e4f3469f..3379fb9374f 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -1280,6 +1280,17 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
code = e.args[0]
self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
if __name__ == '__main__':
import unittest
unittest.TestProgram()
--
2.14.3
From 920621aa792a511576dc6b007ba2c92bf6c229b7 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:10 +1300
Subject: [PATCH 25/42] ldb: Allow tests to operate on ldb_mdb after new
restrictions
The tests were working fine on ldb_mdb but only in the mode without
the GUID index. This mode is not OK for production so has been banned
but this means we need to rework the tests to set an objectGUID on
each record.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++---------------
1 file changed, 121 insertions(+), 64 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 9a2997f5b48..bb2c918e9f6 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -77,6 +77,10 @@ class SimpleLdb(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+ try:
+ self.ldb.add(self.index)
+ except AttributeError:
+ pass
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -165,6 +169,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo1")
m["b"] = [b"a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
self.assertRaises(ldb.LdbError, lambda: l.delete(m.dn, ["search_options:1:2"]))
l.delete(m.dn)
@@ -177,6 +182,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo3")
m["b"] = ["a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
self.assertTrue(ldb.Dn(l, "dc=foo3") in l)
@@ -205,6 +211,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -245,6 +252,7 @@ class SimpleLdb(LdbBaseTest):
m1 = ldb.Message()
m1.dn = ldb.Dn(l, "dc=foo4")
m1["bla"] = b"bla"
+ m1["objectUUID"] = b"0123456789abcdef"
l.add(m1)
try:
s = l.search_iterator()
@@ -261,6 +269,7 @@ class SimpleLdb(LdbBaseTest):
m2 = ldb.Message()
m2.dn = ldb.Dn(l, "dc=foo5")
m2["bla"] = b"bla"
+ m2["objectUUID"] = b"0123456789abcdee"
l.add(m2)
s = l.search_iterator()
@@ -317,6 +326,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = "bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -335,7 +345,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": b"bla"}
+ "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -346,7 +357,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_text(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": "bla"}
+ "bla": "bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -356,7 +368,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_string_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": "dc=foo6", "bla": b"bla"}
+ m = {"dn": "dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -366,7 +379,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_bytes_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": b"dc=foo6", "bla": b"bla"}
+ m = {"dn": b"dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -379,6 +393,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo7")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -392,6 +407,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo8")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
self.assertEqual(len(l.search()), 1)
@@ -406,14 +422,17 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(0, len(l.search()))
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=empty")
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search()
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["blah"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -423,6 +442,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual([b"1234"], list(rm["bla"]))
@@ -434,7 +454,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -446,6 +467,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual(["1234"], list(rm.text["bla"]))
@@ -457,7 +479,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -469,6 +492,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -477,7 +501,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -487,6 +511,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -495,7 +520,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -505,6 +530,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m["bla"] = [b"1234", b"456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -513,7 +539,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"789"], list(rm["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -525,6 +551,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m.text["bla"] = ["1234", "456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -533,7 +560,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["789"], list(rm.text["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -545,6 +572,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -553,7 +581,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -571,6 +599,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -579,7 +608,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -597,6 +626,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo9"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
l.transaction_commit()
l.delete(m.dn)
@@ -606,6 +636,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo10"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdee"
l.add(m)
l.transaction_cancel()
self.assertEqual(0, len(l.search(ldb.Dn(l, "dc=foo10"))))
@@ -625,6 +656,7 @@ class SimpleLdb(LdbBaseTest):
"cN" : b"LDAPtestUSER",
"givenname" : b"ldap",
"displayname" : b"foo\0bar",
+ "objectUUID" : b"0123456789abcdef"
})
res = l.search(expression="(dn=dc=somedn)")
self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
@@ -638,6 +670,10 @@ class SimpleLdbLmdb(SimpleLdb):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SimpleLdbLmdb, self).setUp()
def tearDown(self):
@@ -659,6 +695,10 @@ class SearchTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
self.l.add({"dn": "@ATTRIBUTES",
"DC": "CASE_INSENSITIVE"})
@@ -670,7 +710,7 @@ class SearchTests(LdbBaseTest):
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
- "objectUUID": b"0123456789abcddf"})
+ "objectUUID": b"0123456789abcdef"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
"name": b"Admins",
"x": "z", "y": "a",
@@ -1072,6 +1112,9 @@ class SearchTestsLmdb(SearchTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SearchTestsLmdb, self).setUp()
def tearDown(self):
@@ -1129,12 +1172,12 @@ class GUIDIndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedSearchTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDXGUID = True
self.IDXONE = True
@@ -1143,15 +1186,14 @@ class GUIDIndexedDNFilterSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedDNFilterSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
@@ -1159,16 +1201,14 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
"""Test searches using the index including @IDXONE, to ensure
the index doesn't break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDAndOneLevelIndexedSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
self.IDXONE = True
@@ -1218,6 +1258,11 @@ class AddModifyTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
@@ -1346,23 +1391,15 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
-class AddModifyTestsLmdb(AddModifyTests):
-
- def setUp(self):
- self.prefix = MDB_PREFIX
- super(AddModifyTestsLmdb, self).setUp()
-
- def tearDown(self):
- super(AddModifyTestsLmdb, self).tearDown()
-
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ if not hasattr(self, 'index'):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
+ "@IDXONE": [b"1"]}
super(IndexedAddModifyTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
- "@IDXONE": [b"1"]})
def test_duplicate_GUID(self):
try:
@@ -1436,14 +1473,12 @@ class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedAddModifyTests, self).setUp()
- indexlist = {"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
- m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
- self.l.modify(m)
class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
@@ -2260,19 +2295,36 @@ class LdbResultTests(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.l = ldb.Ldb(self.url(), flags=self.flags())
- self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
- self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
- self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
- self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1"})
- self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2"})
- self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3"})
- self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4"})
- self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5"})
- self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6"})
- self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7"})
- self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8"})
- self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9"})
- self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10"})
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+ self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org",
+ "objectUUID": b"0123456789abcde0"})
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users",
+ "objectUUID": b"0123456789abcde2"})
+ self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1",
+ "objectUUID": b"0123456789abcde3"})
+ self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2",
+ "objectUUID": b"0123456789abcde4"})
+ self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3",
+ "objectUUID": b"0123456789abcde5"})
+ self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4",
+ "objectUUID": b"0123456789abcde6"})
+ self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5",
+ "objectUUID": b"0123456789abcde7"})
+ self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6",
+ "objectUUID": b"0123456789abcde8"})
+ self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7",
+ "objectUUID": b"0123456789abcde9"})
+ self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8",
+ "objectUUID": b"0123456789abcdea"})
+ self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9",
+ "objectUUID": b"0123456789abcdeb"})
+ self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10",
+ "objectUUID": b"0123456789abcdec"})
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -2373,7 +2425,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2444,7 +2497,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2508,6 +2562,9 @@ class LdbResultTestsLmdb(LdbResultTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(LdbResultTestsLmdb, self).setUp()
def tearDown(self):
--
2.14.3
From 0775b29636f4d994bf7bf36fa8e110440735051f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 26/42] ldb_mdb: Apply LMDB key length restrictions at
key-value layer
We need to enforce the GUID index mode so end-users do not get a supprise
in mid-operation and we enforce a max key length of 511 so that the
index key trunctation is done correctly.
Otherwise the DB will appear to work until a very long key (DN or index)
is used, after which it will be sad.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 +++++-
lib/ldb/ldb_tdb/ldb_tdb.c | 17 ++++-
lib/ldb/tests/ldb_lmdb_test.c | 166 +++++++++++++++++++++++++++++++++++++++++-
3 files changed, 200 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 711700c2697..f05d1201e05 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -30,6 +30,8 @@
#define MDB_URL_PREFIX "mdb://"
#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
#define MEGABYTE (1024*1024)
#define GIGABYTE (1024*1024*1024)
@@ -641,7 +643,8 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
{
int ret;
unsigned int mdb_flags;
-
+ int lmdb_max_key_length;
+
if (flags & LDB_FLG_DONT_CREATE_DB) {
struct stat st;
if (stat(path, &st) != 0) {
@@ -695,6 +698,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
+ lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+ /* This will never happen, but if it does make sure to freak out */
+ if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+ talloc_free(lmdb);
+ return ldb_operr(ldb);
+ }
+
return LDB_SUCCESS;
}
@@ -746,6 +757,15 @@ int lmdb_connect(struct ldb_context *ldb,
if (flags & LDB_FLG_RDONLY) {
ltdb->read_only = true;
}
+
+ /*
+ * This maximum length becomes encoded in the index values so
+ * must never change even if LMDB starts to allow longer keys.
+ * The override option is max_key_len_for_self_test, and is
+ * used for testing only.
+ */
+ ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1d06566aa30..b1158241ef9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -651,6 +651,15 @@ static int ltdb_add(struct ltdb_context *ctx)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret = LDB_SUCCESS;
+ if (ltdb->max_key_length != 0 &&
+ ltdb->cache->GUID_index_attribute == NULL &&
+ !ldb_dn_is_special(req->op.add.message->dn)) {
+ ldb_set_errstring(ldb_module_get_ctx(module),
+ "Must operate ldb_mdb in GUID "
+ "index mode, but " LTDB_IDXGUID " not set.");
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ret = ltdb_check_special_dn(module, req->op.add.message);
if (ret != LDB_SUCCESS) {
return ret;
@@ -2149,7 +2158,13 @@ int init_store(struct ltdb_private *ltdb,
*_module = module;
/*
- * Set the maximum key length
+ * Set or override the maximum key length
+ *
+ * The ldb_mdb code will have set this to 511, but our tests
+ * set this even smaller (to make the tests more practical).
+ *
+ * This must only be used for the selftest as the length
+ * becomes encoded in the index keys.
*/
{
const char *len_str =
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index 2902eaf0533..fd82867a275 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -135,12 +135,22 @@ static int ldbtest_setup(void **state)
{
struct ldbtest_ctx *test_ctx;
int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
ldbtest_noconn_setup((void **) &test_ctx);
ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
*state = test_ctx;
return 0;
}
@@ -170,7 +180,8 @@ static void test_ldb_add_key_len_gt_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -184,8 +195,11 @@ static void test_ldb_add_key_len_gt_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
- assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+ assert_int_equal(ret, LDB_SUCCESS);
talloc_free(tmp_ctx);
}
@@ -207,7 +221,8 @@ static void test_ldb_add_key_len_eq_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -220,12 +235,145 @@ static void test_ldb_add_key_len_eq_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
talloc_free(tmp_ctx);
}
+static int ldbtest_setup_noguid(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static void test_ldb_add_special_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_special_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_dn_no_guid_mode(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
+
+ talloc_free(tmp_ctx);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -237,6 +385,18 @@ int main(int argc, const char **argv)
test_ldb_add_key_len_gt_max,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_eq_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_gt_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_dn_no_guid_mode,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.14.3
From 90d9a96b1479a59f94391a2764a3d26f69d28b98 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 27/42] ldb_mdb: Wrap mdb_env_open
Wrap mdb_env_open to ensure that we only have one MDB_env opened per
database in each process
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 155 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 121 insertions(+), 34 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f05d1201e05..4ab3b3f26d9 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -157,6 +157,9 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -212,6 +215,10 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -628,73 +635,154 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
trans_finished(lmdb, ltx);
ltx = lmdb_private_trans_head(lmdb);
}
-
- mdb_env_close(lmdb->env);
lmdb->env = NULL;
return 0;
}
-static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
- struct ldb_context *ldb,
- const char *path,
- unsigned int flags,
- struct lmdb_private *lmdb)
+struct mdb_env_wrap {
+ struct mdb_env_wrap *next, *prev;
+ dev_t device;
+ ino_t inode;
+ MDB_env *env;
+ int pid;
+};
+
+static struct mdb_env_wrap *mdb_list;
+
+/* destroy the last connection to an mdb */
+static int mdb_env_wrap_destructor(struct mdb_env_wrap *w)
{
+ mdb_env_close(w->env);
+ DLIST_REMOVE(mdb_list, w);
+ return 0;
+}
+
+static int lmdb_open_env(TALLOC_CTX *mem_ctx,
+ MDB_env **env,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags) {
int ret;
- unsigned int mdb_flags;
- int lmdb_max_key_length;
-
- if (flags & LDB_FLG_DONT_CREATE_DB) {
- struct stat st;
- if (stat(path, &st) != 0) {
- return LDB_ERR_UNAVAILABLE;
+ unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ /*
+ * MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+
+ struct mdb_env_wrap *w;
+ struct stat st;
+
+ if (stat(path, &st) == 0) {
+ for (w=mdb_list;w;w=w->next) {
+ if (st.st_dev == w->device && st.st_ino == w->inode) {
+ /*
+ * We must have only one MDB_env per process
+ */
+ if (!talloc_reference(mem_ctx, w)) {
+ return ldb_oom(ldb);
+ }
+ *env = w->env;
+ return LDB_SUCCESS;
+ }
}
}
- ret = mdb_env_create(&lmdb->env);
+ w = talloc(mem_ctx, struct mdb_env_wrap);
+ if (w == NULL) {
+ return ldb_oom(ldb);
+ }
+
+ ret = mdb_env_create(env);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
- /* Close when lmdb is released */
- talloc_set_destructor(lmdb, lmdb_pvt_destructor);
-
- ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ /*
+ * Currently we set a 16Gb maximum database size
+ */
+ ret = mdb_env_set_mapsize(*env, 16LL * GIGABYTE);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not open MDB environment %s: %s\n",
path,
mdb_strerror(ret));
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
- mdb_env_set_maxreaders(lmdb->env, 100000);
- /* MDB_NOSUBDIR implies there is a separate file called path and a
- * separate lockfile called path-lock
+ mdb_env_set_maxreaders(*env, 100000);
+ /*
+ * As we ensure that there is only one MDB_env open per database per
+ * process. We can not use the MDB_RDONLY flag, as another ldb may be
+ * opened in read write mode
*/
- mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
- if (flags & LDB_FLG_RDONLY) {
- mdb_flags |= MDB_RDONLY;
- }
if (flags & LDB_FLG_NOSYNC) {
mdb_flags |= MDB_NOSYNC;
}
- ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+ ret = mdb_env_open(*env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
- talloc_free(lmdb);
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
+ if (stat(path, &st) != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not stat %s:\n",
+ path);
+ TALLOC_FREE(w);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ w->env = *env;
+ w->device = st.st_dev;
+ w->inode = st.st_ino;
+
+ talloc_set_destructor(w, mdb_env_wrap_destructor);
+
+ DLIST_ADD(mdb_list, w);
+
+ return LDB_SUCCESS;
+
+}
+
+static int lmdb_pvt_open(struct lmdb_private *lmdb,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags)
+{
+ int ret;
+ int lmdb_max_key_length;
+
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
+ }
+
+ ret = lmdb_open_env(lmdb, &lmdb->env, ldb, path, flags);
+ if (ret != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
@@ -702,7 +790,6 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* This will never happen, but if it does make sure to freak out */
if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
- talloc_free(lmdb);
return ldb_operr(ldb);
}
@@ -739,17 +826,17 @@ int lmdb_connect(struct ldb_context *ldb,
return LDB_ERR_OPERATIONS_ERROR;
}
- lmdb = talloc_zero(ldb, struct lmdb_private);
+ lmdb = talloc_zero(ltdb, struct lmdb_private);
if (lmdb == NULL) {
TALLOC_FREE(ltdb);
- ldb_oom(ldb);
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_oom(ldb);
}
lmdb->ldb = ldb;
ltdb->kv_ops = &lmdb_key_value_ops;
- ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ ret = lmdb_pvt_open(lmdb, ldb, path, flags);
if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(ltdb);
return ret;
}
--
2.14.3
From 21fbe646c2dc697c59cbe9cebab618f533309e2c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 8 Mar 2018 16:47:59 +1300
Subject: [PATCH 28/42] ldb_mdb: Tests for wrap open
Tests to ensure that the mdb_env wrapping code correctly handles
multiple ldb's point to the same physical database file.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 234 ++++++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 5 +-
2 files changed, 237 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 1340f5efa23..916ece8be87 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -38,6 +38,12 @@
#define TEST_BE DEFAULT_BE
#endif /* TEST_BE */
+#ifdef TEST_LMDB
+#include "lmdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+#endif
+
struct ldbtest_ctx {
struct tevent_context *ev;
struct ldb_context *ldb;
@@ -3820,6 +3826,224 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
}
}
+#ifdef TEST_LMDB
+static int test_ldb_multiple_connections_callback(struct ldb_request *req,
+ struct ldb_reply *ares)
+{
+ int ret;
+ int pipes[2];
+ char buf[2];
+ int pid, child_pid;
+ int wstatus;
+
+ switch (ares->type) {
+ case LDB_REPLY_ENTRY:
+ break;
+
+ case LDB_REPLY_REFERRAL:
+ return LDB_SUCCESS;
+
+ case LDB_REPLY_DONE:
+ return ldb_request_done(req, LDB_SUCCESS);
+ }
+
+ {
+ /*
+ * We open a new ldb on an ldb that is already open and
+ * then close it.
+ *
+ * If the multiple connection wrapping is correct the
+ * underlying MDB_env will be left open and we should see
+ * an active reader in the child we fork next
+ */
+ struct ldb_context *ldb = NULL;
+ struct tevent_context *ev = NULL;
+ TALLOC_CTX *mem_ctx = talloc_new(NULL);
+
+ ev = tevent_context_init(mem_ctx);
+ assert_non_null(ev);
+
+ ldb = ldb_init(mem_ctx, ev);
+ assert_non_null(ldb);
+
+ ret = ldb_connect(ldb, TEST_BE"://apitest.ldb" , 0, NULL);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+ TALLOC_FREE(ldb);
+ TALLOC_FREE(mem_ctx);
+ }
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct MDB_env *env = NULL;
+ struct MDB_envinfo stat;
+
+ /*
+ * Check that there are exactly two readers on the MDB file
+ * backing the ldb.
+ *
+ */
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_create returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_open(env,
+ "apitest.ldb",
+ MDB_NOSUBDIR | MDB_NOTLS,
+ 0644);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_open returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_info(env, &stat);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_info returned (%d)",
+ ret);
+ exit(ret);
+ }
+ if (stat.me_numreaders != 2) {
+ print_error(__location__
+ " Incorrect number of readers (%d)",
+ stat.me_numreaders);
+ exit(LDB_ERR_CONSTRAINT_VIOLATION);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+ return LDB_SUCCESS;
+
+}
+
+static void test_ldb_close_with_multiple_connections(void **state)
+{
+ struct search_test_ctx *search_test_ctx = NULL;
+ struct ldb_dn *search_dn = NULL;
+ struct ldb_request *req = NULL;
+ int ret = 0;
+
+ search_test_ctx = talloc_get_type_abort(*state, struct search_test_ctx);
+ assert_non_null(search_test_ctx);
+
+ search_dn = ldb_dn_new_fmt(search_test_ctx,
+ search_test_ctx->ldb_test_ctx->ldb,
+ "cn=test_search_cn,"
+ "dc=search_test_entry");
+ assert_non_null(search_dn);
+
+ /*
+ * The search just needs to call DONE, we don't care about the
+ * contents of the search for this test
+ */
+ ret = ldb_build_search_req(&req,
+ search_test_ctx->ldb_test_ctx->ldb,
+ search_test_ctx,
+ search_dn,
+ LDB_SCOPE_SUBTREE,
+ "(&(!(filterAttr=*))"
+ "(cn=test_search_cn))",
+ NULL,
+ NULL,
+ NULL,
+ test_ldb_multiple_connections_callback,
+ NULL);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_request(search_test_ctx->ldb_test_ctx->ldb, req);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+ assert_int_equal(ret, 0);
+}
+
+static struct MDB_env* get_mdb_env(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct MDB_env *env = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ lmdb = ltdb->lmdb_private;
+ assert_non_null(lmdb);
+
+ env = lmdb->env;
+ assert_non_null(env);
+
+ return env;
+}
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ struct MDB_env *env3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+ env3 = get_mdb_env(ldb3);
+
+ assert_ptr_equal(env1, env2);
+ assert_ptr_equal(env1, env3);
+
+}
+#endif
int main(int argc, const char **argv)
{
@@ -3984,6 +4208,16 @@ int main(int argc, const char **argv)
test_ldb_talloc_destructor_transaction_cleanup,
ldbtest_setup,
ldbtest_teardown),
+#ifdef TEST_LMDB
+ cmocka_unit_test_setup_teardown(
+ test_ldb_close_with_multiple_connections,
+ ldb_search_test_setup,
+ ldb_search_test_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
+#endif
};
return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 577203beefb..7623c52d1b6 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -443,8 +443,9 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
- cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
- deps='cmocka ldb',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 ' +
+ '-DTEST_LMDB=1',
+ deps='cmocka ldb lmdb',
install=False)
bld.SAMBA_BINARY('ldb_lmdb_test',
--
2.14.3
From 8f2c0226355746f2ca03784f9978ee36ec651f5a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 21 Mar 2018 11:38:22 +1300
Subject: [PATCH 29/42] ldb_mdb: handle EBADE from mdb_env_open
Under some circumstances mdb_env_open returns EBADE, we treat this as
indicating the file is not a valid lmdb format file.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 4ab3b3f26d9..17f30b8ca49 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -42,6 +42,7 @@ int ldb_mdb_err_map(int lmdb_err)
return LDB_SUCCESS;
case EIO:
return LDB_ERR_OPERATIONS_ERROR;
+ case EBADE:
case MDB_INCOMPATIBLE:
case MDB_CORRUPTED:
case MDB_INVALID:
@@ -777,7 +778,7 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
/* Close when lmdb is released */
--
2.14.3
From 1369ef1f9dc254d97a628033432cae49a8c8fda1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:14:09 +1300
Subject: [PATCH 30/42] ldb index tests: test large index key value
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 44 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index fd82867a275..92715fdf96f 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -204,6 +204,46 @@ static void test_ldb_add_key_len_gt_max(void **state)
talloc_free(tmp_ctx);
}
+static void test_ldb_add_key_len_2x_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = 2 * LMDB_MAX_KEY_SIZE;
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
static void test_ldb_add_key_len_eq_max(void **state)
{
int ret;
@@ -385,6 +425,10 @@ int main(int argc, const char **argv)
test_ldb_add_key_len_gt_max,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_2x_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
cmocka_unit_test_setup_teardown(
test_ldb_add_special_key_len_eq_max,
ldbtest_setup_noguid,
--
2.14.3
From 185a9300f74d41c70ba96f8e1223a3d4ec3a4e1d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 12:43:25 +1300
Subject: [PATCH 31/42] ldb test: close pipes to stop forked tests failing on
failure
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 916ece8be87..9f62ba57041 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3881,6 +3881,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
if (child_pid == 0) {
struct MDB_env *env = NULL;
struct MDB_envinfo stat;
+ close(pipes[0]);
/*
* Check that there are exactly two readers on the MDB file
@@ -3929,6 +3930,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
}
exit(LDB_SUCCESS);
}
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
--
2.14.3
From e8dfc11748abe7d41a882f835805810ff7f88669 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 32/42] ldb_mdb: prevent MDB_env reuse across forks
MDB_env's may not be reused accross forks. Check the pid that the lmdb
structure was created by, and return an error if it is being used by a
different process.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 33 +++++-
lib/ldb/tests/ldb_mod_op_test.c | 233 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 264 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 17f30b8ca49..d73b7abc190 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -434,6 +434,18 @@ static int lmdb_lock_read(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct lmdb_private *lmdb = ltdb->lmdb_private;
+ pid_t pid = getpid();
+
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
lmdb->error = MDB_SUCCESS;
if (ltdb->in_transaction == 0 &&
@@ -473,12 +485,25 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
struct lmdb_trans *ltx;
struct lmdb_trans *ltx_head;
MDB_txn *tx_parent;
+ pid_t pid = getpid();
ltx = talloc_zero(lmdb, struct lmdb_trans);
if (ltx == NULL) {
return ldb_oom(lmdb->ldb);
}
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opened by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
+
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
@@ -646,7 +671,7 @@ struct mdb_env_wrap {
dev_t device;
ino_t inode;
MDB_env *env;
- int pid;
+ pid_t pid;
};
static struct mdb_env_wrap *mdb_list;
@@ -673,10 +698,13 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
struct mdb_env_wrap *w;
struct stat st;
+ pid_t pid = getpid();
if (stat(path, &st) == 0) {
for (w=mdb_list;w;w=w->next) {
- if (st.st_dev == w->device && st.st_ino == w->inode) {
+ if (st.st_dev == w->device &&
+ st.st_ino == w->inode &&
+ pid == w->pid) {
/*
* We must have only one MDB_env per process
*/
@@ -747,6 +775,7 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
w->env = *env;
w->device = st.st_dev;
w->inode = st.st_ino;
+ w->pid = pid;
talloc_set_destructor(w, mdb_env_wrap_destructor);
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 9f62ba57041..b8204685691 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -4045,6 +4045,227 @@ static void test_multiple_opens(void **state)
assert_ptr_equal(env1, env3);
}
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ env3 = get_mdb_env(ldb3);
+ if (env1 != env2) {
+ print_error(__location__": env1 != env2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (env1 == env3) {
+ print_error(__location__": env1 == env3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_transaction_start_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ close(pipes[0]);
+ ret = ldb_transaction_start(ldb1);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_transaction_start "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_lock_read_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_search "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+ {
+ /*
+ * Ensure that the search actually succeeds on the opening
+ * pid
+ */
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ assert_int_equal(0, ret);
+ }
+}
#endif
int main(int argc, const char **argv)
@@ -4219,6 +4440,18 @@ int main(int argc, const char **argv)
test_multiple_opens,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_start_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_lock_read_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
#endif
};
--
2.14.3
From d73d49561385fac3a0724440db8fa892224bba11 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:25:28 +1300
Subject: [PATCH 33/42] ldb: make backends expose if there is an active
transaction
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 37 +++++++++++++++++++++----------------
lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
3 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index d73b7abc190..1dbb8713ac8 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -92,6 +92,11 @@ static int lmdb_error_at(struct ldb_context *ldb,
return ldb_err;
}
+
+static bool lmdb_transaction_active(struct ltdb_private *ltdb) {
+ return ltdb->lmdb_private->txlist != NULL;
+}
+
static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
{
if (ltx == NULL) {
@@ -582,23 +587,23 @@ static bool lmdb_changed(struct ltdb_private *ltdb)
return true;
}
-
static struct kv_db_ops lmdb_key_value_ops = {
- .store = lmdb_store,
- .delete = lmdb_delete,
- .iterate = lmdb_traverse_fn,
- .update_in_iterate = lmdb_update_in_iterate,
- .fetch_and_parse = lmdb_parse_record,
- .lock_read = lmdb_lock_read,
- .unlock_read = lmdb_unlock_read,
- .begin_write = lmdb_transaction_start,
- .prepare_write = lmdb_transaction_prepare_commit,
- .finish_write = lmdb_transaction_commit,
- .abort_write = lmdb_transaction_cancel,
- .error = lmdb_error,
- .errorstr = lmdb_errorstr,
- .name = lmdb_name,
- .has_changed = lmdb_changed,
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+ .transaction_active = lmdb_transaction_active,
};
static const char *lmdb_get_path(const char *url)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index b1158241ef9..66cd6cb9ff7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1924,6 +1924,10 @@ static bool ltdb_tdb_changed(struct ltdb_private *ltdb)
return has_changed;
}
+static bool ltdb_transaction_active(struct ltdb_private *ltdb) {
+ return tdb_transaction_active(ltdb->tdb);
+}
+
static const struct kv_db_ops key_value_ops = {
.store = ltdb_tdb_store,
.delete = ltdb_tdb_delete,
@@ -1940,6 +1944,7 @@ static const struct kv_db_ops key_value_ops = {
.errorstr = ltdb_errorstr,
.name = ltdb_tdb_name,
.has_changed = ltdb_tdb_changed,
+ .transaction_active = ltdb_transaction_active,
};
static void ltdb_callback(struct tevent_context *ev,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 7d92804a8e9..59fd06aac40 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -29,6 +29,7 @@ struct kv_db_ops {
const char * (*errorstr)(struct ltdb_private *ltdb);
const char * (*name)(struct ltdb_private *ltdb);
bool (*has_changed)(struct ltdb_private *ltdb);
+ bool (*transaction_active)(struct ltdb_private *ltdb);
};
/* this private structure is used by the ltdb backend in the
--
2.14.3
From 288c69a90c44bf6e3109a5391f1d4544fd7d075b Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 5 Jan 2018 09:28:54 +1300
Subject: [PATCH 34/42] tests: Replace some references to tdb with ldb://
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/tests/encrypted_secrets.py | 2 +-
source4/torture/drs/python/samba_tool_drs.py | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/python/samba/tests/encrypted_secrets.py b/python/samba/tests/encrypted_secrets.py
index 3b6934f0ccb..114f3b91108 100644
--- a/python/samba/tests/encrypted_secrets.py
+++ b/python/samba/tests/encrypted_secrets.py
@@ -54,7 +54,7 @@ class EncryptedSecretsTests(TestCase):
backend_subpath = os.path.join("sam.ldb.d",
backend_filename)
backend_path = self.lp.private_path(backend_subpath)
- backenddb = ldb.Ldb(backend_path)
+ backenddb = ldb.Ldb("ldb://" + backend_path, flags=ldb.FLG_DONT_CREATE_DB)
dn = "CN=Administrator,CN=Users,%s" % basedn
diff --git a/source4/torture/drs/python/samba_tool_drs.py b/source4/torture/drs/python/samba_tool_drs.py
index fcc681dc9e4..502c8096603 100644
--- a/source4/torture/drs/python/samba_tool_drs.py
+++ b/source4/torture/drs/python/samba_tool_drs.py
@@ -282,7 +282,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
@@ -291,7 +291,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertEqual(ds_name, server_ds_name)
self.assertEqual(ldap_service_name, server_ldap_service_name)
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
def get_krbtgt_pw():
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
@@ -346,13 +346,13 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
config_nc_name = ldb_rootdse["configurationNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
self.assertIsNotNone(krbtgt_pw)
@@ -381,7 +381,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertRaises(samba.tests.BlackboxProcessError, demote_self)
# While we have this cloned, try demoting the other server on the clone
- out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H %s/private/sam.ldb"
+ out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H ldb://%s/private/sam.ldb"
% (self.dc2,
self.tempdir))
--
2.14.3
From 38a7d1788dbcf55404143a093fc2890f4b300b7b Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 12:40:01 +1300
Subject: [PATCH 35/42] tests/dlz_bind9: support for multiple db types
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
source4/torture/dns/dlz_bind9.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/torture/dns/dlz_bind9.c b/source4/torture/dns/dlz_bind9.c
index 2234e7a23f6..42b104e070c 100644
--- a/source4/torture/dns/dlz_bind9.c
+++ b/source4/torture/dns/dlz_bind9.c
@@ -58,7 +58,7 @@ static char *test_dlz_bind9_binddns_dir(struct torture_context *tctx,
const char *file)
{
return talloc_asprintf(tctx,
- "%s/%s",
+ "ldb://%s/%s",
lpcfg_binddns_dir(tctx->lp_ctx),
file);
}
--
2.14.3
From 467df48def80bfe4ee85e35925595b548789a959 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:27:10 +1300
Subject: [PATCH 36/42] ldb_tdb: Do not make search or DB modifications without
a lock
The ldb_cache startup code would previously not take a read lock
nor a sufficiently wide write transaction.
The new code takes a read lock, and if it needs to write takes a
write lock (transaction) and re-reads before continuing.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 50 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 38 insertions(+), 12 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 4790bcd7e53..0d16452b913 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -386,6 +386,7 @@ int ltdb_cache_load(struct ldb_module *module)
uint64_t seq;
struct ldb_message *baseinfo = NULL, *options = NULL;
const struct ldb_schema_attribute *a;
+ bool have_write_txn = false;
int r;
ldb = ldb_module_get_ctx(module);
@@ -406,29 +407,38 @@ int ltdb_cache_load(struct ldb_module *module)
baseinfo_dn = ldb_dn_new(baseinfo, ldb, LTDB_BASEINFO);
if (baseinfo_dn == NULL) goto failed;
+ r = ltdb->kv_ops->lock_read(module);
+ if (r != LDB_SUCCESS) {
+ goto failed;
+ }
r= ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0);
if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
- goto failed;
+ goto failed_and_unlock;
}
-
+
/* possibly initialise the baseinfo */
if (r == LDB_ERR_NO_SUCH_OBJECT) {
+ /* Give up the read lock, try again with a write lock */
+ r = ltdb->kv_ops->unlock_read(module);
+ if (r != LDB_SUCCESS) {
+ goto failed;
+ }
+
if (ltdb->kv_ops->begin_write(ltdb) != 0) {
goto failed;
}
+ have_write_txn = true;
+
/* error handling for ltdb_baseinfo_init() is by
looking for the record again. */
ltdb_baseinfo_init(module);
- if (ltdb->kv_ops->finish_write(ltdb) != 0) {
- goto failed;
- }
-
if (ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0) != LDB_SUCCESS) {
- goto failed;
+ goto failed_and_unlock;
}
+
}
/* Ignore the result, and update the sequence number */
@@ -443,16 +453,17 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb->sequence_number = seq;
/* Read an interpret database options */
+
options = ldb_msg_new(ltdb->cache);
- if (options == NULL) goto failed;
+ if (options == NULL) goto failed_and_unlock;
options_dn = ldb_dn_new(options, ldb, LTDB_OPTIONS);
- if (options_dn == NULL) goto failed;
+ if (options_dn == NULL) goto failed_and_unlock;
r= ltdb_search_dn1(module, options_dn, options, 0);
talloc_free(options_dn);
if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
- goto failed;
+ goto failed_and_unlock;
}
/* set flags if they do exist */
@@ -479,7 +490,7 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb_attributes_unload(module);
if (ltdb_index_load(module, ltdb) == -1) {
- goto failed;
+ goto failed_and_unlock;
}
/*
@@ -488,7 +499,7 @@ int ltdb_cache_load(struct ldb_module *module)
* partition module.
*/
if (ltdb_attributes_load(module) == -1) {
- goto failed;
+ goto failed_and_unlock;
}
ltdb->GUID_index_syntax = NULL;
@@ -503,10 +514,25 @@ int ltdb_cache_load(struct ldb_module *module)
}
done:
+ if (have_write_txn) {
+ if (ltdb->kv_ops->finish_write(ltdb) != 0) {
+ goto failed;
+ }
+ } else {
+ ltdb->kv_ops->unlock_read(module);
+ }
+
talloc_free(options);
talloc_free(baseinfo);
return 0;
+failed_and_unlock:
+ if (have_write_txn) {
+ ltdb->kv_ops->abort_write(ltdb);
+ } else {
+ ltdb->kv_ops->unlock_read(module);
+ }
+
failed:
talloc_free(options);
talloc_free(baseinfo);
--
2.14.3
From 3e945bfea0a8d1ce7078e6b775c7492f33b4414c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:29:25 +1300
Subject: [PATCH 37/42] ldb_mdb: Remove implicit read lock and remove
transaction counter
The way to know if we are in a transaction is if there is a non-NULL
transaction handle.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 51 +++++++++++++++--------------------------------
1 file changed, 16 insertions(+), 35 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 1dbb8713ac8..f607afee648 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -130,26 +130,21 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
return ltx;
}
+
static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
{
MDB_txn *txn;
- if (lmdb->read_txn != NULL) {
- return lmdb->read_txn;
- }
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
- if (txn == NULL) {
- int ret;
- ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
- if (ret != 0) {
- lmdb->error = ret;
- ldb_asprintf_errstring(lmdb->ldb,
- "%s failed: %s\n", __FUNCTION__,
- mdb_strerror(ret));
- }
- lmdb->read_txn = txn;
+ if (txn != NULL) {
+ return txn;
+ }
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
}
- return txn;
+ lmdb->error = MDB_BAD_TXN;
+ ldb_set_errstring(lmdb->ldb, __location__":No active transaction\n");
+ return NULL;
}
static int lmdb_store(struct ltdb_private *ltdb,
@@ -194,10 +189,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_val value;
lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
if (lmdb->error != MDB_SUCCESS) {
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
mdb_flags = 0;
@@ -302,11 +293,6 @@ done:
mdb_cursor_close(cursor);
}
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
-
if (lmdb->error != MDB_SUCCESS) {
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
@@ -410,10 +396,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
if (lmdb->error != MDB_SUCCESS) {
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
if (lmdb->error == MDB_NOTFOUND) {
return LDB_ERR_NO_SUCH_OBJECT;
}
@@ -425,11 +407,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- /* We created a read transaction, commit it */
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return parser(key, data, ctx);
}
@@ -453,7 +430,7 @@ static int lmdb_lock_read(struct ldb_module *module)
}
lmdb->error = MDB_SUCCESS;
- if (ltdb->in_transaction == 0 &&
+ if (lmdb_transaction_active(ltdb) == false &&
ltdb->read_lock_count == 0) {
lmdb->error = mdb_txn_begin(lmdb->env,
NULL,
@@ -473,7 +450,7 @@ static int lmdb_unlock_read(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ if (lmdb_transaction_active(ltdb) == false && ltdb->read_lock_count == 1) {
struct lmdb_private *lmdb = ltdb->lmdb_private;
mdb_txn_commit(lmdb->read_txn);
lmdb->read_txn = NULL;
@@ -492,6 +469,11 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
MDB_txn *tx_parent;
pid_t pid = getpid();
+ /* Do not take out the transaction lock on a read-only DB */
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ltx = talloc_zero(lmdb, struct lmdb_trans);
if (ltx == NULL) {
return ldb_oom(lmdb->ldb);
@@ -508,7 +490,6 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
return LDB_ERR_PROTOCOL_ERROR;
}
-
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
--
2.14.3
From 20f09b54944d98c626680a6ba37c3e819e49c516 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:28:18 +1300
Subject: [PATCH 38/42] ldb_tdb: Disallow TDB nested transactions and use
tdb_transaction_active()
This avoids keeping a counter, which can be error-prone.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 22 ++++++++++------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 -
2 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 66cd6cb9ff7..babee775008 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -101,7 +101,7 @@ static int ltdb_lock_read(struct ldb_module *module)
int tdb_ret = 0;
int ret;
- if (ltdb->in_transaction == 0 &&
+ if (tdb_transaction_active(ltdb->tdb) == false &&
ltdb->read_lock_count == 0) {
tdb_ret = tdb_lockall_read(ltdb->tdb);
}
@@ -128,7 +128,7 @@ static int ltdb_unlock_read(struct ldb_module *module)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
tdb_unlockall_read(ltdb->tdb);
ltdb->read_lock_count--;
return 0;
@@ -379,7 +379,7 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
/* only allow modifies inside a transaction, otherwise the
* ldb is unsafe */
- if (ltdb->in_transaction == 0) {
+ if (ltdb->kv_ops->transaction_active(ltdb) == false) {
ldb_set_errstring(ldb_module_get_ctx(module), "ltdb modify without transaction");
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1478,7 +1478,6 @@ static int ltdb_start_trans(struct ldb_module *module)
return ltdb->kv_ops->error(ltdb);
}
- ltdb->in_transaction++;
ltdb_index_transaction_start(module);
@@ -1499,8 +1498,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction != 1) {
- return LDB_SUCCESS;
+ if (!ltdb->kv_ops->transaction_active(ltdb)) {
+ ldb_set_errstring(ldb_module_get_ctx(module),
+ "ltdb_prepare_commit() called "
+ "without transaction active");
+ return LDB_ERR_OPERATIONS_ERROR;
}
/*
@@ -1524,13 +1526,11 @@ static int ltdb_prepare_commit(struct ldb_module *module)
ret = ltdb_index_transaction_commit(module);
if (ret != LDB_SUCCESS) {
ltdb->kv_ops->abort_write(ltdb);
- ltdb->in_transaction--;
return ret;
}
if (ltdb->kv_ops->prepare_write(ltdb) != 0) {
ret = ltdb->kv_ops->error(ltdb);
- ltdb->in_transaction--;
ldb_debug_set(ldb_module_get_ctx(module),
LDB_DEBUG_FATAL,
"Failure during "
@@ -1558,7 +1558,6 @@ static int ltdb_end_trans(struct ldb_module *module)
}
}
- ltdb->in_transaction--;
ltdb->prepared_commit = false;
if (ltdb->kv_ops->finish_write(ltdb) != 0) {
@@ -1578,7 +1577,6 @@ static int ltdb_del_trans(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- ltdb->in_transaction--;
if (ltdb_index_transaction_cancel(module) != 0) {
ltdb->kv_ops->abort_write(ltdb);
@@ -1808,7 +1806,7 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
.ctx = ctx,
.ltdb = ltdb
};
- if (ltdb->in_transaction != 0) {
+ if (tdb_transaction_active(ltdb->tdb)) {
return tdb_traverse(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
} else {
return tdb_traverse_read(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
@@ -2213,7 +2211,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
path = url;
}
- tdb_flags = TDB_DEFAULT | TDB_SEQNUM;
+ tdb_flags = TDB_DEFAULT | TDB_SEQNUM | TDB_DISALLOW_NESTING;
/* check for the 'nosync' option */
if (flags & LDB_FLG_NOSYNC) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 59fd06aac40..9c3f8d89d8d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -54,7 +54,6 @@ struct ltdb_private {
const char *GUID_index_dn_component;
} *cache;
- int in_transaction;
bool check_base;
bool disallow_dn_filter;
--
2.14.3
From 63d693758eb5f78aa9b6be7593272f06d96fbfe8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 4 Apr 2018 17:21:30 +1200
Subject: [PATCH 39/42] ldb_tdb: Disallow reads without a transaction or read
lock
This will ensure we match LMDB behaviour and avoid a repeat of the per-record locking
issues (compared with full DB locking) we had before Samba 4.7.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index babee775008..4f31210c5fc 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1899,6 +1899,11 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
};
int ret;
+ if (tdb_transaction_active(ltdb->tdb) == false &&
+ ltdb->read_lock_count == 0) {
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
&kv_ctx);
if (ret == 0) {
--
2.14.3
From ba840fe6e5171a59e203d1db163914a495e42c6b Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:14:10 +1300
Subject: [PATCH 40/42] ldb tests: api ensure database correctly populated
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index bb2c918e9f6..1222d122199 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -1316,6 +1316,19 @@ class AddModifyTests(LdbBaseTest):
"name": b"Admins",
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde2"})
+
+ res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde1)")
+ self.assertEqual(len(res2), 1)
+ self.assertEqual(str(res2[0].dn), "OU=DUP,DC=SAMBA,DC=ORG")
+
+ res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde2)")
+ self.assertEqual(len(res3), 1)
+ self.assertEqual(str(res3[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
try:
self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
"OU=DUP2,DC=SAMBA,DC=ORG")
--
2.14.3
From ca81e0c3416106c8452d1454ec05512e8dd55bb6 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:15:12 +1300
Subject: [PATCH 41/42] ldb tests: add cmocka tests of kv operation
interactions with transactions
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_kv_ops_test.c | 184 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 184 insertions(+)
diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
index 837b139d3be..8c6d5a32916 100644
--- a/lib/ldb/tests/ldb_kv_ops_test.c
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -268,6 +268,63 @@ static void test_add_get(void **state)
talloc_free(tmp_ctx);
}
+/*
+ * Test that attempts to read data without a read transaction fail.
+ */
+static void test_read_outside_transaction(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ * Note there is no read transaction active
+ */
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
/*
* Test that data can be deleted from the kv store
*/
@@ -730,6 +787,125 @@ static void test_iterate(void **state)
TALLOC_FREE(tmp_ctx);
}
+struct update_context {
+ struct ldb_context* ldb;
+ int visits[NUM_RECS];
+};
+
+static int update_fn(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val data,
+ void *ctx) {
+
+ struct ldb_val new_key;
+ struct ldb_module *module = NULL;
+ struct update_context *context =NULL;
+ int ret = LDB_SUCCESS;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(ltdb);
+ assert_non_null(tmp_ctx);
+
+ context = talloc_get_type_abort(ctx, struct update_context);
+
+ module = talloc_zero(tmp_ctx, struct ldb_module);
+ module->ldb = context->ldb;
+
+ if (strncmp("key ", (char *) key.data, 4) == 0) {
+ int i = strtol((char *) &key.data[4], NULL, 10);
+ context->visits[i]++;
+ new_key.data = talloc_memdup(tmp_ctx, key.data, key.length);
+ new_key.length = key.length;
+ new_key.data[0] = 'K';
+
+ ret = ltdb->kv_ops->update_in_iterate(ltdb,
+ key,
+ new_key,
+ data,
+ &module);
+ }
+ TALLOC_FREE(tmp_ctx);
+ return ret;
+}
+
+/*
+ * Test that update_in_iterate behaves as expected.
+ */
+static void test_update_in_iterate(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ int i;
+ struct update_context *context = NULL;
+
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ context = talloc_zero(tmp_ctx, struct update_context);
+ assert_non_null(context);
+ context->ldb = test_ctx->ldb;
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the records
+ */
+ for (i = 0; i < NUM_RECS; i++) {
+ struct ldb_val key;
+ struct ldb_val rec;
+ int flags = 0;
+
+ key.data = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+ key.length = strlen((char *)key.data) + 1;
+
+ rec.data = (uint8_t *) talloc_asprintf(tmp_ctx,
+ "data for record (%04d)",
+ i);
+ rec.length = strlen((char *)rec.data) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(key.data);
+ TALLOC_FREE(rec.data);
+ }
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now iterate over the kv store and ensure that all the
+ * records are visited.
+ */
+
+ /*
+ * Needs to be done inside a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ ret = ltdb->kv_ops->iterate(ltdb, update_fn, context);
+ for (i = 0; i < NUM_RECS; i++) {
+ assert_int_equal(1, context->visits[i]);
+ }
+
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(tmp_ctx);
+}
+
/*
* Ensure that writes are not visible until the transaction has been
* committed.
@@ -1369,6 +1545,10 @@ int main(int argc, const char **argv)
test_transaction_abort_delete,
setup,
teardown),
+ cmocka_unit_test_setup_teardown(
+ test_read_outside_transaction,
+ setup,
+ teardown),
cmocka_unit_test_setup_teardown(
test_write_outside_transaction,
setup,
@@ -1381,6 +1561,10 @@ int main(int argc, const char **argv)
test_iterate,
setup,
teardown),
+ cmocka_unit_test_setup_teardown(
+ test_update_in_iterate,
+ setup,
+ teardown),
cmocka_unit_test_setup_teardown(
test_write_transaction_isolation,
setup,
--
2.14.3
From 4a5eb1c574e4db30d1db4cb002453391bbf94992 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 20 Mar 2018 12:58:02 +1300
Subject: [PATCH 42/42] ldb: Release ldb 1.4.0
* New LMDB backend
* Comprehensive tests for index behaviour
* Enforce transactions for writes
* Enforce read lock use for all reads
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ABI/ldb-1.4.0.sigs | 279 ++++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.4.0.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 284 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
diff --git a/lib/ldb/ABI/ldb-1.4.0.sigs b/lib/ldb/ABI/ldb-1.4.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.4.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.4.0.sigs b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 7623c52d1b6..4a09211ff67 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.3.2'
+VERSION = '1.4.0'
blddir = 'bin'
--
2.14.3
More information about the samba-technical
mailing list