[PATCH] GUID index for LDB
Andrew Bartlett
abartlet at samba.org
Wed Sep 13 05:16:01 UTC 2017
On Tue, 2017-09-12 at 23:10 +1200, Andrew Bartlett via samba-technical
wrote:
> On Tue, 2017-09-12 at 13:21 +1200, Andrew Bartlett via samba-
> technical
> wrote:
> >
> > In my current branch I've added some incremental output during the
> > re-
> > index.
>
> Attached is my current branch. A recent version of branch passed a
> full autobuild (multiple times), and I trust this will also.
>
> Please review, as the only outstanding things as far as I know are:
> - any required tests for the downgrade script
> - the dns wildcard issue
Attached is the current patch set. Please review so I can get this in
to master soon.
Thanks,
Andrew Bartlett
--
Andrew Bartlett
https://samba.org/~abartlet/
Authentication Developer, Samba Team https://samba.org
Samba Development and Support, Catalyst IT
https://catalyst.net.nz/services/samba
-------------- next part --------------
From b4cf70da7b97ff8c1d73579783eb78b1d9894d69 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 16:13:06 +1200
Subject: [PATCH 01/94] selftest: Check re-opening sam.ldb corrects the
@ATTRIBUTES and @INDEXLIST
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/tests/dsdb_schema_attributes.py | 53 ++++++++++++++++++++++++++++
selftest/knownfail.d/dsdb_schema_attributes | 2 ++
2 files changed, 55 insertions(+)
create mode 100644 selftest/knownfail.d/dsdb_schema_attributes
diff --git a/python/samba/tests/dsdb_schema_attributes.py b/python/samba/tests/dsdb_schema_attributes.py
index df6c8bb11b2..2bebbb569f2 100644
--- a/python/samba/tests/dsdb_schema_attributes.py
+++ b/python/samba/tests/dsdb_schema_attributes.py
@@ -173,3 +173,56 @@ systemOnly: FALSE
self.assertIn(attr_ldap_name, [str(x) for x in idx_res[0]["@IDXATTR"]])
self.assertIn(attr_ldap_name2, [str(x) for x in idx_res[0]["@IDXATTR"]])
+
+ def test_modify_at_attributes(self):
+ m = {"dn": "@ATTRIBUTES",
+ "@TEST_EXTRA": ["HIDDEN"]
+ }
+
+ msg = ldb.Message.from_dict(self.samdb, m, ldb.FLAG_MOD_ADD)
+ self.samdb.modify(msg)
+
+ res = self.samdb.search(base="@ATTRIBUTES", scope=ldb.SCOPE_BASE,
+ attrs=["@TEST_EXTRA"])
+ self.assertEquals(len(res), 1)
+ self.assertEquals(str(res[0].dn), "@ATTRIBUTES")
+ self.assertEquals(len(res[0]), 1)
+ self.assertTrue("@TEST_EXTRA" in res[0])
+ self.assertEquals(len(res[0]["@TEST_EXTRA"]), 1)
+ self.assertEquals(res[0]["@TEST_EXTRA"][0], "HIDDEN")
+
+ samdb2 = samba.tests.connect_samdb(self.lp.samdb_url())
+
+ res = self.samdb.search(base="@ATTRIBUTES", scope=ldb.SCOPE_BASE,
+ attrs=["@TEST_EXTRA"])
+ self.assertEquals(len(res), 1)
+ self.assertEquals(str(res[0].dn), "@ATTRIBUTES")
+ self.assertEquals(len(res[0]), 0)
+ self.assertFalse("@TEST_EXTRA" in res[0])
+
+
+ def test_modify_at_indexlist(self):
+ m = {"dn": "@INDEXLIST",
+ "@TEST_EXTRA": ["1"]
+ }
+
+ msg = ldb.Message.from_dict(self.samdb, m, ldb.FLAG_MOD_ADD)
+ self.samdb.modify(msg)
+
+ res = self.samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE,
+ attrs=["@TEST_EXTRA"])
+ self.assertEquals(len(res), 1)
+ self.assertEquals(str(res[0].dn), "@INDEXLIST")
+ self.assertEquals(len(res[0]), 1)
+ self.assertTrue("@TEST_EXTRA" in res[0])
+ self.assertEquals(len(res[0]["@TEST_EXTRA"]), 1)
+ self.assertEquals(res[0]["@TEST_EXTRA"][0], "1")
+
+ samdb2 = samba.tests.connect_samdb(self.lp.samdb_url())
+
+ res = self.samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE,
+ attrs=["@TEST_EXTRA"])
+ self.assertEquals(len(res), 1)
+ self.assertEquals(str(res[0].dn), "@INDEXLIST")
+ self.assertEquals(len(res[0]), 0)
+ self.assertFalse("@TEST_EXTRA" in res[0])
diff --git a/selftest/knownfail.d/dsdb_schema_attributes b/selftest/knownfail.d/dsdb_schema_attributes
new file mode 100644
index 00000000000..4242212707c
--- /dev/null
+++ b/selftest/knownfail.d/dsdb_schema_attributes
@@ -0,0 +1,2 @@
+^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_indexlist
+^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_attributes
\ No newline at end of file
--
2.11.0
From 28cd235e23c158132ab25c4a2f174fa6bf0a4310 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 15:22:23 +1200
Subject: [PATCH 02/94] schema: Rework dsdb_schema_set_indices_and_attributes()
db operations
Commit ec9b1e881c3eef503d6b4b311594113acf7d47d8 did not fully fix this.
There is no value in using dsdb_replace(), we are under the read lock
and replace just confuses things further.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=13025
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
selftest/knownfail.d/dsdb_schema_attributes | 2 --
source4/dsdb/schema/schema_set.c | 17 +++++++++--------
2 files changed, 9 insertions(+), 10 deletions(-)
delete mode 100644 selftest/knownfail.d/dsdb_schema_attributes
diff --git a/selftest/knownfail.d/dsdb_schema_attributes b/selftest/knownfail.d/dsdb_schema_attributes
deleted file mode 100644
index 4242212707c..00000000000
--- a/selftest/knownfail.d/dsdb_schema_attributes
+++ /dev/null
@@ -1,2 +0,0 @@
-^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_indexlist
-^samba.tests.dsdb_schema_attributes.samba.tests.dsdb_schema_attributes.SchemaAttributesTestCase.test_modify_at_attributes
\ No newline at end of file
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index cfd320b2407..8141e323e50 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -175,11 +175,11 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
}
if (mod_msg->num_elements > 0) {
/*
- * Do the replace with the constructed message,
- * to avoid needing a lock between this search
- * and the replace
+ * Do the replace with the difference, as we
+ * are under the read lock and we wish to do a
+ * delete of any removed/renamed attributes
*/
- ret = dsdb_replace(ldb, msg, 0);
+ ret = dsdb_modify(ldb, mod_msg, 0);
}
talloc_free(mod_msg);
}
@@ -235,12 +235,13 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
* @SAMBA_FEATURES_SUPPORTED
*/
} else if (mod_msg->num_elements > 0) {
+
/*
- * Do the replace with the constructed message,
- * to avoid needing a lock between this search
- * and the replace
+ * Do the replace with the difference, as we
+ * are under the read lock and we wish to do a
+ * delete of any removed/renamed attributes
*/
- ret = dsdb_replace(ldb, msg_idx, 0);
+ ret = dsdb_modify(ldb, mod_msg, 0);
}
talloc_free(mod_msg);
}
--
2.11.0
From 735fbc55df0180dc6801f8846466555579ed4e73 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 12 Sep 2017 14:17:35 +1200
Subject: [PATCH 03/94] selftest: reindex in dbcheck-oldrelease after modifying
the backend DB
Modifying the backend DB is not a supported operation, but helps us create test
situations.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
testprogs/blackbox/dbcheck-oldrelease.sh | 3 +++
1 file changed, 3 insertions(+)
diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh
index ecab0030d68..a525f547bc9 100755
--- a/testprogs/blackbox/dbcheck-oldrelease.sh
+++ b/testprogs/blackbox/dbcheck-oldrelease.sh
@@ -404,6 +404,9 @@ if [ -d $release_dir ]; then
testit_expect_failure "dbcheck2" dbcheck2
testit "dbcheck_clean2" dbcheck_clean2
testit "rm_deleted_objects" rm_deleted_objects
+ # We must re-index again because rm_deleted_objects went behind
+ # the back of the main sam.ldb.
+ testit "reindex2" reindex
testit_expect_failure "dbcheck3" dbcheck3
testit "dbcheck_clean3" dbcheck_clean3
testit "check_expected_after_deleted_objects" check_expected_after_deleted_objects
--
2.11.0
From c4e25cc74137d962202c3fd799d3726525d9437e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 6 Sep 2017 16:24:35 +1200
Subject: [PATCH 04/94] repl_meta_data: Show failing replicated entry in error
code
This re-work of our LDIF printing avoids some of the privacy issue from
printing the full LDIF at level 4, while showing the entry that actually fails.
Instead, with e3988f8f74f4a11e8f26a548e0a33d20f4e863f7 we now print the DN
only at level 4, then the full message at 8.
With this patch on failure, we print the redacted failing message at 5.
While all of the DRS replication data is potentially sensitive
the passwords are most sensitive, and are now not printed unencrypted.
This discourages users from sending the full failing trace, as the
last entry is much more likely the issue.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 36 +++++++++++++++++++++++--
1 file changed, 34 insertions(+), 2 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
index 8f123de7dbe..d2c2084acb1 100644
--- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
+++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c
@@ -563,9 +563,41 @@ static int replmd_op_callback(struct ldb_request *req, struct ldb_reply *ares)
}
if (ares->error != LDB_SUCCESS) {
- DEBUG(5,("%s failure. Error is: %s\n", __FUNCTION__, ldb_strerror(ares->error)));
+ struct GUID_txt_buf guid_txt;
+ struct ldb_message *msg = NULL;
+ char *s = NULL;
+
+ if (ac->apply_mode == false) {
+ DBG_NOTICE("Originating update failure. Error is: %s\n",
+ ldb_strerror(ares->error));
+ return ldb_module_done(ac->req, controls,
+ ares->response, ares->error);
+ }
+
+ msg = ac->objs->objects[ac->index_current].msg;
+ /*
+ * Set at DBG_NOTICE as once these start to happe, they
+ * will happen a lot until resolved, due to repeated
+ * replication. The caller will probably print the
+ * ldb error string anyway.
+ */
+ DBG_NOTICE("DRS replication apply failure for %s. Error is: %s\n",
+ ldb_dn_get_linearized(msg->dn),
+ ldb_strerror(ares->error));
+
+ s = ldb_ldif_message_redacted_string(ldb_module_get_ctx(ac->module),
+ ac,
+ LDB_CHANGETYPE_ADD,
+ msg);
+
+ DBG_INFO("Failing DRS %s replication message was %s:\n%s\n",
+ ac->search_msg == NULL ? "ADD" : "MODIFY",
+ GUID_buf_string(&ac->objs->objects[ac->index_current].object_guid,
+ &guid_txt),
+ s);
+ talloc_free(s);
return ldb_module_done(ac->req, controls,
- ares->response, ares->error);
+ ares->response, ares->error);
}
if (ares->type != LDB_REPLY_DONE) {
--
2.11.0
From 851f7c8e1ce3d3a538f19760e46a5ef6c8d1b5bf Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 11:48:46 +1200
Subject: [PATCH 05/94] s4-dnsserver: Always encode user-supplied names when
looking up DNS records
BUG: https://bugzilla.samba.org/show_bug.cgi?id=12994
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
---
source4/rpc_server/dnsserver/dcerpc_dnsserver.c | 15 ++++++++++++---
source4/rpc_server/dnsserver/dnsdb.c | 7 +++++--
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/source4/rpc_server/dnsserver/dcerpc_dnsserver.c b/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
index 286da18346b..120d4b9a3d7 100644
--- a/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
+++ b/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
@@ -1674,10 +1674,13 @@ static WERROR dnsserver_enumerate_root_records(struct dnsserver_state *dsstate,
/* Add any additional records */
if (select_flag & DNS_RPC_VIEW_ADDITIONAL_DATA) {
for (i=0; i<add_count; i++) {
+ char *encoded_name
+ = ldb_binary_encode_string(tmp_ctx,
+ add_names[i]);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))",
- add_names[i]);
+ encoded_name);
if (ret != LDB_SUCCESS || res->count == 0) {
talloc_free(res);
continue;
@@ -1744,10 +1747,12 @@ static WERROR dnsserver_enumerate_records(struct dnsserver_state *dsstate,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(!(dNSTombstoned=TRUE)))");
} else {
+ char *encoded_name
+ = ldb_binary_encode_string(tmp_ctx, name);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(|(name=%s)(name=*.%s))(!(dNSTombstoned=TRUE)))",
- name, name);
+ encoded_name, encoded_name);
}
if (ret != LDB_SUCCESS) {
talloc_free(tmp_ctx);
@@ -1818,11 +1823,15 @@ static WERROR dnsserver_enumerate_records(struct dnsserver_state *dsstate,
/* Search all the available zones for additional name */
for (z2 = dsstate->zones; z2; z2 = z2->next) {
+ char *encoded_name;
name = dns_split_node_name(tmp_ctx, add_names[i], z2->name);
+ encoded_name
+ = ldb_binary_encode_string(tmp_ctx,
+ name);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z2->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))",
- name);
+ encoded_name);
talloc_free(name);
if (ret != LDB_SUCCESS) {
continue;
diff --git a/source4/rpc_server/dnsserver/dnsdb.c b/source4/rpc_server/dnsserver/dnsdb.c
index da37878ce2c..f937de95949 100644
--- a/source4/rpc_server/dnsserver/dnsdb.c
+++ b/source4/rpc_server/dnsserver/dnsdb.c
@@ -364,10 +364,12 @@ WERROR dnsserver_db_add_empty_node(TALLOC_CTX *mem_ctx,
const char * const attrs[] = { "name", NULL };
struct ldb_result *res;
struct ldb_dn *dn;
+ char *encoded_name = ldb_binary_encode_string(mem_ctx, name);
int ret;
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_BASE, attrs,
- "(&(objectClass=dnsNode)(name=%s))", name);
+ "(&(objectClass=dnsNode)(name=%s))",
+ encoded_name);
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
@@ -642,7 +644,8 @@ WERROR dnsserver_db_delete_record(TALLOC_CTX *mem_ctx,
}
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_ONELEVEL, attrs,
- "(&(objectClass=dnsNode)(name=%s))", name);
+ "(&(objectClass=dnsNode)(name=%s))",
+ ldb_binary_encode_string(mem_ctx, name));
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
--
2.11.0
From b190a69aef6a86e485734c9e353931634afddefa Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 14:19:22 +1200
Subject: [PATCH 06/94] s4-dnsserver: Check for too many DNS results
If we had this check in when the wildcard DNS tests were written, we would have
noticed that the name needed to be escaped (see previous commit).
BUG: https://bugzilla.samba.org/show_bug.cgi?id=12994
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
---
source4/rpc_server/dnsserver/dnsdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/source4/rpc_server/dnsserver/dnsdb.c b/source4/rpc_server/dnsserver/dnsdb.c
index f937de95949..3ba475938a8 100644
--- a/source4/rpc_server/dnsserver/dnsdb.c
+++ b/source4/rpc_server/dnsserver/dnsdb.c
@@ -653,6 +653,9 @@ WERROR dnsserver_db_delete_record(TALLOC_CTX *mem_ctx,
if (res->count == 0) {
return WERR_DNS_ERROR_RECORD_DOES_NOT_EXIST;
}
+ if (res->count > 1) {
+ return WERR_DNS_ERROR_RCODE_SERVER_FAILURE;
+ }
el = ldb_msg_find_element(res->msgs[0], "dnsRecord");
if (el == NULL || el->num_values == 0) {
--
2.11.0
From 814464e638f2249ef9fd5e05b018d3b26b11e0cf Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 30 Aug 2017 15:30:04 +1200
Subject: [PATCH 07/94] s4-provision: Ensure the dummy main-domain DB used for
DLZ has an @INDEXLIST
The other databases are created from copies of the main provision, but this one
is not, so did not previously get a valid @INDEXLIST.
This is important as otherwise we will not correctly notice support for
the GUID index or new DSDB features in @SAMBA_DSDB as this is gated
on seeing @SAMBA_FEATURES_SUPPORTED in @INDEXLIST.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/provision/sambadns.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index d4cb93a89ea..fce72adc533 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -809,6 +809,10 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
"DESCRIPTOR" : descr})
setup_add_ldif(dom_ldb,
setup_path("provision_basedn_options.ldif"), None)
+
+ # We need the dummy main-domain DB to have the correct @INDEXLIST
+ index_res = samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE)
+ dom_ldb.add(index_res[0])
except:
logger.error(
"Failed to setup database for BIND, AD based DNS cannot be used")
--
2.11.0
From 8c0d0e3bd5129179be027279698995a0e6fe76ff Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 28 Aug 2017 14:55:00 +1200
Subject: [PATCH 08/94] ldb_tdb: Make ldb_match_message() available to ldb_tdb
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/common/ldb_match.c | 22 +++++++++++++++-------
lib/ldb/include/ldb_private.h | 19 +++++++++++++++++++
2 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/lib/ldb/common/ldb_match.c b/lib/ldb/common/ldb_match.c
index 1415fac59f9..25fe3f9c21b 100644
--- a/lib/ldb/common/ldb_match.c
+++ b/lib/ldb/common/ldb_match.c
@@ -515,17 +515,18 @@ static int ldb_match_extended(struct ldb_context *ldb,
}
/*
- return 0 if the given parse tree matches the given message. Assumes
- the message is in sorted order
+ Check if a particular message will match the given filter
- return 1 if it matches, and 0 if it doesn't match
+ set *matched to true if it matches, false otherwise
+
+ returns LDB_SUCCESS or an error
this is a recursive function, and does short-circuit evaluation
*/
-static int ldb_match_message(struct ldb_context *ldb,
- const struct ldb_message *msg,
- const struct ldb_parse_tree *tree,
- enum ldb_scope scope, bool *matched)
+int ldb_match_message(struct ldb_context *ldb,
+ const struct ldb_message *msg,
+ const struct ldb_parse_tree *tree,
+ enum ldb_scope scope, bool *matched)
{
unsigned int i;
int ret;
@@ -587,6 +588,13 @@ static int ldb_match_message(struct ldb_context *ldb,
return LDB_ERR_INAPPROPRIATE_MATCHING;
}
+/*
+ return 0 if the given parse tree matches the given message. Assumes
+ the message is in sorted order
+
+ return 1 if it matches, and 0 if it doesn't match
+*/
+
int ldb_match_msg(struct ldb_context *ldb,
const struct ldb_message *msg,
const struct ldb_parse_tree *tree,
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index ae7ec3cfd0d..ab215411a9a 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -287,5 +287,24 @@ int ldb_msg_find_duplicate_val(struct ldb_context *ldb,
const struct ldb_message_element *el,
struct ldb_val **duplicate,
uint32_t options);
+/**
+ Check if a particular message will match the given filter
+
+ \param ldb an ldb context
+ \param msg the message to be checked
+ \param tree the filter tree to check against
+ \param scope the scope to match against
+ (to avoid matching special DNs except on a base search)
+ \param matched a pointer to a boolean set true if it matches,
+ false otherwise
+
+ returns LDB_SUCCESS or an error
+
+ \note this is a recursive function, and does short-circuit evaluation
+ */
+int ldb_match_message(struct ldb_context *ldb,
+ const struct ldb_message *msg,
+ const struct ldb_parse_tree *tree,
+ enum ldb_scope scope, bool *matched);
#endif
--
2.11.0
From f44feee38438a7f878753eb0b81b9272fa0aeb7b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 17:21:28 +1200
Subject: [PATCH 09/94] ldb_tdb: Add helper function
ltdb_search_and_return_base()
This avoids an extra DB lookup for the base, when that is the only
record we will return, and avoids going into the index code for
a base search, as that won't work for special DNs once the GUID
index mode is enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 100 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 95 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index a6c408a53ce..117f6858a17 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -501,6 +501,81 @@ static int ltdb_search_full(struct ltdb_context *ctx)
return ctx->error;
}
+static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
+ struct ltdb_context *ctx)
+{
+ struct ldb_message *msg, *filtered_msg;
+ struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
+ int ret;
+ bool matched;
+
+ msg = ldb_msg_new(ctx);
+ if (!msg) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ret = ltdb_search_dn1(ctx->module, ctx->base, msg,
+ LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
+ LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
+
+ if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ if (ltdb->check_base == false) {
+ /*
+ * In this case, we are done, as no base
+ * checking is allowed in this DB
+ */
+ talloc_free(msg);
+ return LDB_SUCCESS;
+ }
+ ldb_asprintf_errstring(ldb,
+ "No such Base DN: %s",
+ ldb_dn_get_linearized(ctx->base));
+ }
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ret;
+ }
+
+
+ /*
+ * We use this, not ldb_match_msg_error() as we know
+ * we matched on the scope BASE, as we just fetched
+ * the base DN
+ */
+
+ ret = ldb_match_message(ldb, msg,
+ ctx->tree,
+ ctx->scope,
+ &matched);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ret;
+ }
+ if (!matched) {
+ talloc_free(msg);
+ return LDB_SUCCESS;
+ }
+
+ /* filter the attributes that the user wants */
+ ret = ltdb_filter_attrs(ctx, msg, ctx->attrs, &filtered_msg);
+
+ talloc_free(msg);
+
+ if (ret == -1) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ldb_module_send_entry(ctx->req, filtered_msg, NULL);
+ if (ret != LDB_SUCCESS) {
+ /* Regardless of success or failure, the msg
+ * is the callbacks responsiblity, and should
+ * not be talloc_free()'ed */
+ ctx->request_terminated = true;
+ return ret;
+ }
+
+ return LDB_SUCCESS;
+}
+
/*
search the database with a LDAP-like expression.
choses a search method
@@ -532,6 +607,11 @@ int ltdb_search(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
+ ctx->tree = req->op.search.tree;
+ ctx->scope = req->op.search.scope;
+ ctx->base = req->op.search.base;
+ ctx->attrs = req->op.search.attrs;
+
if ((req->op.search.base == NULL) || (ldb_dn_is_null(req->op.search.base) == true)) {
/* Check what we should do with a NULL dn */
@@ -559,6 +639,21 @@ int ltdb_search(struct ltdb_context *ctx)
ldb_dn_get_linearized(req->op.search.base));
ret = LDB_ERR_INVALID_DN_SYNTAX;
+ } else if (req->op.search.scope == LDB_SCOPE_BASE) {
+
+ /*
+ * If we are LDB_SCOPE_BASE, do just one search and
+ * return early. This is critical to ensure we do not
+ * go into the index code for special DNs, as that
+ * will try to look up an index record for a special
+ * record (which doesn't exist).
+ */
+ ret = ltdb_search_and_return_base(ltdb, ctx);
+
+ ltdb_unlock_read(module);
+
+ return ret;
+
} else if (ltdb->check_base) {
/* This database has been marked as 'checkBaseOnSearch', so do a spot check of the base dn */
ret = ltdb_search_base(module, req->op.search.base);
@@ -574,11 +669,6 @@ int ltdb_search(struct ltdb_context *ctx)
ret = LDB_SUCCESS;
}
- ctx->tree = req->op.search.tree;
- ctx->scope = req->op.search.scope;
- ctx->base = req->op.search.base;
- ctx->attrs = req->op.search.attrs;
-
if (ret == LDB_SUCCESS) {
uint32_t match_count = 0;
--
2.11.0
From 307a09424b93af4c5624d30a32f7e0f67bc75458 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 14:31:18 +1200
Subject: [PATCH 10/94] ldb_tdb: provide ldb_key_dn() and ldb_key_msg()
This will in time allow us to generate a TDB key from
the msg, eg from an objectGUID.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
lib/ldb/ldb_tdb/ldb_search.c | 4 ++--
lib/ldb/ldb_tdb/ldb_tdb.c | 23 ++++++++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.h | 3 ++-
4 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3510dd9b1bc..dec7bcca1cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1625,7 +1625,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
/* check if the DN key has changed, perhaps due to the
case insensitivity of an element changing */
- key2 = ltdb_key(module, msg->dn);
+ key2 = ltdb_key_dn(module, msg->dn);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 117f6858a17..4d3c38a0b08 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -125,7 +125,7 @@ static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
}
/* form the key */
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -214,7 +214,7 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
};
/* form the key */
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OPERATIONS_ERROR;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index ccad8168a6e..0a239d49dd5 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -163,7 +163,7 @@ bool ltdb_key_is_record(TDB_DATA key)
note that the key for a record can depend on whether the
dn refers to a case sensitive index record or not
*/
-TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
TDB_DATA key;
@@ -210,6 +210,19 @@ failed:
}
/*
+ form a TDB_DATA for a record key
+ caller frees
+
+ note that the key for a record can depend on whether the
+ dn refers to a case sensitive index record or not
+*/
+TDB_DATA ltdb_key_msg(struct ldb_module *module,
+ const struct ldb_message *msg)
+{
+ return ltdb_key_dn(module, msg->dn);
+}
+
+/*
check special dn's have valid attributes
currently only @ATTRIBUTES is checked
*/
@@ -297,7 +310,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
- tdb_key = ltdb_key(module, msg->dn);
+ tdb_key = ltdb_key_msg(module, msg);
if (tdb_key.dptr == NULL) {
return LDB_ERR_OTHER;
}
@@ -465,7 +478,7 @@ int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
TDB_DATA tdb_key;
int ret;
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OTHER;
}
@@ -1093,13 +1106,13 @@ static int ltdb_rename(struct ltdb_context *ctx)
/* We need to, before changing the DB, check if the new DN
* exists, so we can return this error to the caller with an
* unmodified DB */
- tdb_key = ltdb_key(module, req->op.rename.newdn);
+ tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key_old = ltdb_key(module, req->op.rename.olddn);
+ tdb_key_old = ltdb_key_dn(module, req->op.rename.olddn);
if (!tdb_key_old.dptr) {
talloc_free(msg);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index a391606dd16..52e707e9d85 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -115,7 +115,8 @@ int ltdb_unlock_read(struct ldb_module *module);
* index, the old DN index and a possible future ID=
*/
bool ltdb_key_is_record(TDB_DATA key);
-TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
--
2.11.0
From 9a400620fb67b199ed42332207e29ed519e0ed1b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:01:12 +1200
Subject: [PATCH 11/94] ldb_tdb: Use ltdb_key_msg() in re_index()
This will allow changing to a GUID tdb key in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index dec7bcca1cc..613fc1b4d6d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1623,9 +1623,10 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
return -1;
}
- /* check if the DN key has changed, perhaps due to the
- case insensitivity of an element changing */
- key2 = ltdb_key_dn(module, msg->dn);
+ /* check if the DN key has changed, perhaps due to the case
+ insensitivity of an element changing, or a change from DN
+ to GUID keys */
+ key2 = ltdb_key_msg(module, msg);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
--
2.11.0
From 822e84ba108e8608a03d0888e8cbc606658a8817 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:06:08 +1200
Subject: [PATCH 12/94] ldb_tdb: Add ltdb_search_key()
This allows us to slowly split out the tdb key in the DB from being the DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 41 +++++++++++++++++++++++++++++++----------
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 4d3c38a0b08..bbceead56f0 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -200,25 +200,18 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
+int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
+ struct TDB_DATA tdb_key,
+ struct ldb_message *msg,
unsigned int unpack_flags)
{
- void *data = ldb_module_get_private(module);
- struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
- TDB_DATA tdb_key;
struct ltdb_parse_data_unpack_ctx ctx = {
.msg = msg,
.module = module,
.unpack_flags = unpack_flags
};
- /* form the key */
- tdb_key = ltdb_key_dn(module, dn);
- if (!tdb_key.dptr) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
memset(msg, 0, sizeof(*msg));
msg->num_elements = 0;
@@ -237,6 +230,34 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return ret;
}
+ return LDB_SUCCESS;
+}
+
+/*
+ search the database for a single simple dn, returning all attributes
+ in a single message
+
+ return LDB_ERR_NO_SUCH_OBJECT on record-not-found
+ and LDB_SUCCESS on success
+*/
+int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
+ unsigned int unpack_flags)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ int ret;
+ TDB_DATA tdb_key;
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, dn);
+ if (!tdb_key.dptr) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if ((unpack_flags & LDB_UNPACK_DATA_FLAG_NO_DN) == 0) {
if (!msg->dn) {
msg->dn = ldb_dn_copy(msg, dn);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 52e707e9d85..b4bc330fe92 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -102,6 +102,10 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
+int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
+ struct TDB_DATA tdb_key,
+ struct ldb_message *msg,
+ unsigned int unpack_flags);
int ltdb_filter_attrs(TALLOC_CTX *mem_ctx,
const struct ldb_message *msg, const char * const *attrs,
struct ldb_message **filtered_msg);
--
2.11.0
From c1b8e11c4f1cfe45e64e1321e9833e68a96b02a2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 12:46:57 +1200
Subject: [PATCH 13/94] ldb_tdb: Use a more complete error mapping in
ltdb_search_key()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index bbceead56f0..753eb940048 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -222,10 +222,15 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
talloc_free(tdb_key.dptr);
if (ret == -1) {
- if (tdb_error(ltdb->tdb) == TDB_ERR_NOEXIST) {
- return LDB_ERR_NO_SUCH_OBJECT;
+ ret = ltdb_err_map(tdb_error(ltdb->tdb));
+ if (ret == LDB_SUCCESS) {
+ /*
+ * Just to be sure we don't turn errors
+ * into success
+ */
+ return LDB_ERR_OPERATIONS_ERROR;
}
- return LDB_ERR_OPERATIONS_ERROR;
+ return ret;
} else if (ret != LDB_SUCCESS) {
return ret;
}
--
2.11.0
From 0ab9bfc7fc272322e417060bded176c65c5529ea Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:23:33 +1200
Subject: [PATCH 14/94] ldb_tdb: Provide struct ltdb_private to index routines
This will make it easier to switch the GUID index mode on and off
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 90 ++++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.c | 38 +++++++++++++------
lib/ldb/ldb_tdb/ldb_tdb.h | 16 ++++++--
3 files changed, 99 insertions(+), 45 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 613fc1b4d6d..8fd30cfd736 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -79,7 +79,9 @@ static int dn_list_cmp(const struct ldb_val *v1, const struct ldb_val *v2)
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
comparison with the dn returns -1 if not found
*/
-static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_val *v)
+static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
+ const struct dn_list *list,
+ const struct ldb_val *v)
{
unsigned int i;
for (i=0; i<list->count; i++) {
@@ -94,12 +96,14 @@ static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_va
find a entry in a dn_list. Uses a case sensitive comparison with the dn
returns -1 if not found
*/
-static int ltdb_dn_list_find_str(struct dn_list *list, const char *dn)
+static int ltdb_dn_list_find_str(struct ltdb_private *ltdb,
+ struct dn_list *list,
+ const char *dn)
{
struct ldb_val v;
v.data = discard_const_p(unsigned char, dn);
v.length = strlen(dn);
- return ltdb_dn_list_find_val(list, &v);
+ return ltdb_dn_list_find_val(ltdb, list, &v);
}
/*
@@ -219,7 +223,9 @@ normal_index:
/*
save a dn_list into a full @IDX style record
*/
-static int ltdb_dn_list_store_full(struct ldb_module *module, struct ldb_dn *dn,
+static int ltdb_dn_list_store_full(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct dn_list *list)
{
struct ldb_message *msg;
@@ -274,7 +280,8 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
struct dn_list *list2;
if (ltdb->idxptr == NULL) {
- return ltdb_dn_list_store_full(module, dn, list);
+ return ltdb_dn_list_store_full(module, ltdb,
+ dn, list);
}
if (ltdb->idxptr->itdb == NULL) {
@@ -345,7 +352,8 @@ static int ltdb_index_traverse_store(struct tdb_context *tdb, TDB_DATA key, TDB_
return -1;
}
- ltdb->idxptr->error = ltdb_dn_list_store_full(module, dn, list);
+ ltdb->idxptr->error = ltdb_dn_list_store_full(module, ltdb,
+ dn, list);
talloc_free(dn);
if (ltdb->idxptr->error != 0) {
return -1;
@@ -575,6 +583,7 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
list = list & list2
*/
static bool list_intersect(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
struct dn_list *list, const struct dn_list *list2)
{
struct dn_list *list3;
@@ -622,7 +631,8 @@ static bool list_intersect(struct ldb_context *ldb,
list3->count = 0;
for (i=0;i<list->count;i++) {
- if (ltdb_dn_list_find_val(list2, &list->dn[i]) != -1) {
+ if (ltdb_dn_list_find_val(ltdb, list2,
+ &list->dn[i]) != -1) {
list3->dn[list3->count] = list->dn[i];
list3->count++;
}
@@ -846,7 +856,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
list->dn = list2->dn;
list->count = list2->count;
found = true;
- } else if (!list_intersect(ldb, list, list2)) {
+ } else if (!list_intersect(ldb, ltdb,
+ list, list2)) {
talloc_free(list2);
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -952,7 +963,8 @@ static int ltdb_index_dn(struct ldb_module *module,
filter a candidate dn_list from an indexed search into a set of results
extracting just the given attributes
*/
-static int ltdb_index_filter(const struct dn_list *dn_list,
+static int ltdb_index_filter(struct ltdb_private *ltdb,
+ const struct dn_list *dn_list,
struct ltdb_context *ac,
uint32_t *match_count)
{
@@ -1123,7 +1135,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
break;
}
- ret = ltdb_index_filter(dn_list, ac, match_count);
+ ret = ltdb_index_filter(ltdb, dn_list, ac, match_count);
talloc_free(dn_list);
return ret;
}
@@ -1148,7 +1160,9 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
*
* @return An ldb error code
*/
-static int ltdb_index_add1(struct ldb_module *module, const char *dn,
+static int ltdb_index_add1(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *el, int v_idx)
{
struct ldb_context *ldb;
@@ -1227,12 +1241,15 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
/*
add index entries for one elements in a message
*/
-static int ltdb_index_add_el(struct ldb_module *module, const char *dn,
+static int ltdb_index_add_el(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *el)
{
unsigned int i;
for (i = 0; i < el->num_values; i++) {
- int ret = ltdb_index_add1(module, dn, el, i);
+ int ret = ltdb_index_add1(module, ltdb,
+ dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1244,11 +1261,12 @@ static int ltdb_index_add_el(struct ldb_module *module, const char *dn,
/*
add index entries for all elements in a message
*/
-static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
+static int ltdb_index_add_all(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *elements,
unsigned int num_el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
unsigned int i;
if (dn[0] == '@') {
@@ -1265,7 +1283,7 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
- ret = ltdb_index_add_el(module, dn, &elements[i]);
+ ret = ltdb_index_add_el(module, ltdb, dn, &elements[i]);
if (ret != LDB_SUCCESS) {
struct ldb_context *ldb = ldb_module_get_ctx(module);
ldb_asprintf_errstring(ldb,
@@ -1321,9 +1339,9 @@ static int ltdb_index_onelevel(struct ldb_module *module,
el.num_values = 1;
if (add) {
- ret = ltdb_index_add1(module, dn, &el, 0);
+ ret = ltdb_index_add1(module, ltdb, dn, &el, 0);
} else { /* delete */
- ret = ltdb_index_del_value(module, msg->dn, &el, 0);
+ ret = ltdb_index_del_value(module, ltdb, msg->dn, &el, 0);
}
talloc_free(pdn);
@@ -1335,23 +1353,27 @@ static int ltdb_index_onelevel(struct ldb_module *module,
add the index entries for a new element in a record
The caller guarantees that these element values are not yet indexed
*/
-int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_add_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
if (ldb_dn_is_special(dn)) {
return LDB_SUCCESS;
}
if (!ltdb_is_indexed(module, ltdb, el->name)) {
return LDB_SUCCESS;
}
- return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el);
+ return ltdb_index_add_el(module, ltdb,
+ ldb_dn_get_linearized(dn), el);
}
/*
add the index entries for a new record
*/
-int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
+int ltdb_index_add_new(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg)
{
const char *dn;
int ret;
@@ -1365,7 +1387,8 @@ int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, dn, msg->elements,
+ msg->num_elements);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1377,7 +1400,9 @@ int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
/*
delete an index entry for one message element
*/
-int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_value(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el, unsigned int v_idx)
{
struct ldb_context *ldb;
@@ -1422,7 +1447,7 @@ int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
return ret;
}
- i = ltdb_dn_list_find_str(list, dn_str);
+ i = ltdb_dn_list_find_str(ltdb, list, dn_str);
if (i == -1) {
/* nothing to delete */
talloc_free(dn_key);
@@ -1452,10 +1477,11 @@ int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
delete the index entries for a element
return -1 on failure
*/
-int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
const char *dn_str;
int ret;
unsigned int i;
@@ -1478,7 +1504,7 @@ int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
return LDB_SUCCESS;
}
for (i = 0; i < el->num_values; i++) {
- ret = ltdb_index_del_value(module, dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1512,7 +1538,8 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
}
for (i = 0; i < msg->num_elements; i++) {
- ret = ltdb_index_del_element(module, msg->dn, &msg->elements[i]);
+ ret = ltdb_index_del_element(module, ltdb,
+ msg->dn, &msg->elements[i]);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1678,6 +1705,8 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
struct ldb_context *ldb;
struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
struct ldb_module *module = ctx->module;
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
struct ldb_message *msg;
const char *dn = NULL;
unsigned int nb_elements_in_db;
@@ -1739,7 +1768,8 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
return -1;
}
- ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, dn,
+ msg->elements, msg->num_elements);
if (ret != LDB_SUCCESS) {
ctx->error = ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 0a239d49dd5..f0855ecba9a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -368,6 +368,7 @@ static bool ldb_tdb_single_valued(const struct ldb_schema_attribute *a,
}
static int ltdb_add_internal(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_message *msg,
bool check_single_value)
{
@@ -432,7 +433,7 @@ static int ltdb_add_internal(struct ldb_module *module,
return ret;
}
- ret = ltdb_index_add_new(module, msg);
+ ret = ltdb_index_add_new(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -449,6 +450,8 @@ static int ltdb_add(struct ltdb_context *ctx)
{
struct ldb_module *module = ctx->module;
struct ldb_request *req = ctx->req;
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret = LDB_SUCCESS;
ret = ltdb_check_special_dn(module, req->op.add.message);
@@ -462,7 +465,8 @@ static int ltdb_add(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_add_internal(module, req->op.add.message, true);
+ ret = ltdb_add_internal(module, ltdb,
+ req->op.add.message, true);
return ret;
}
@@ -622,6 +626,7 @@ static int ltdb_msg_add_element(struct ldb_message *msg,
delete all elements having a specified attribute name
*/
static int msg_delete_attribute(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_message *msg, const char *name)
{
unsigned int i;
@@ -634,7 +639,7 @@ static int msg_delete_attribute(struct ldb_module *module,
}
i = el - msg->elements;
- ret = ltdb_index_del_element(module, msg->dn, el);
+ ret = ltdb_index_del_element(module, ltdb, msg->dn, el);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -656,6 +661,7 @@ static int msg_delete_attribute(struct ldb_module *module,
return LDB Error on failure
*/
static int msg_delete_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_message *msg,
const char *name,
const struct ldb_val *val)
@@ -688,10 +694,11 @@ static int msg_delete_element(struct ldb_module *module,
}
if (matched) {
if (el->num_values == 1) {
- return msg_delete_attribute(module, msg, name);
+ return msg_delete_attribute(module,
+ ltdb, msg, name);
}
- ret = ltdb_index_del_value(module, msg->dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg->dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -728,6 +735,8 @@ int ltdb_modify_internal(struct ldb_module *module,
struct ldb_request *req)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct ldb_message *msg2;
unsigned int i, j;
int ret = LDB_SUCCESS, idx;
@@ -813,7 +822,8 @@ int ltdb_modify_internal(struct ldb_module *module,
ret = LDB_ERR_OTHER;
goto done;
}
- ret = ltdb_index_add_element(module, msg2->dn,
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn,
el);
if (ret != LDB_SUCCESS) {
goto done;
@@ -894,7 +904,8 @@ int ltdb_modify_internal(struct ldb_module *module,
el2->values = vals;
el2->num_values += el->num_values;
- ret = ltdb_index_add_element(module, msg2->dn, el);
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -958,7 +969,8 @@ int ltdb_modify_internal(struct ldb_module *module,
}
/* Delete the attribute if it exists in the DB */
- if (msg_delete_attribute(module, msg2,
+ if (msg_delete_attribute(module, ltdb,
+ msg2,
el->name) != 0) {
ret = LDB_ERR_OTHER;
goto done;
@@ -971,7 +983,8 @@ int ltdb_modify_internal(struct ldb_module *module,
goto done;
}
- ret = ltdb_index_add_element(module, msg2->dn, el);
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -987,7 +1000,9 @@ int ltdb_modify_internal(struct ldb_module *module,
if (msg->elements[i].num_values == 0) {
/* Delete the whole attribute */
- ret = msg_delete_attribute(module, msg2,
+ ret = msg_delete_attribute(module,
+ ltdb,
+ msg2,
msg->elements[i].name);
if (ret == LDB_ERR_NO_SUCH_ATTRIBUTE &&
control_permissive) {
@@ -1004,6 +1019,7 @@ int ltdb_modify_internal(struct ldb_module *module,
/* Delete specified values from an attribute */
for (j=0; j < msg->elements[i].num_values; j++) {
ret = msg_delete_element(module,
+ ltdb,
msg2,
msg->elements[i].name,
&msg->elements[i].values[j]);
@@ -1155,7 +1171,7 @@ static int ltdb_rename(struct ltdb_context *ctx)
* deleted attributes. We could go through all elements but that's
* maybe not the most efficient way
*/
- ret = ltdb_add_internal(module, msg, false);
+ ret = ltdb_add_internal(module, ltdb, msg, false);
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index b4bc330fe92..09c3c5441ab 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -82,13 +82,21 @@ int ltdb_check_at_attributes_values(const struct ldb_val *value);
struct ldb_parse_tree;
int ltdb_search_indexed(struct ltdb_context *ctx, uint32_t *);
-int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg);
+int ltdb_index_add_new(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg);
int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg);
-int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el);
-int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_add_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el);
-int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_value(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el, unsigned int v_idx);
int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
--
2.11.0
From 7b7d278c3a849605058824efeda835c06738a135 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:13:48 +1200
Subject: [PATCH 15/94] ldb_tdb: Add GUID_index_attribute to ltdb_private
This will be used to determine if we are in GUID index mode
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 09c3c5441ab..aa8d162fdec 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -20,6 +20,7 @@ struct ltdb_private {
struct ldb_message *indexlist;
bool one_level_indexes;
bool attribute_indexes;
+ const char *GUID_index_attribute;
} *cache;
int in_transaction;
--
2.11.0
From e0c73ebec2849f5dda6cb75f6845a1aac870f42b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:42:25 +1200
Subject: [PATCH 16/94] ldb_tdb: Do not allow a modification of the
GUID_index_attribute (objectGUID)
This would totally break our index scheme if this could be modified.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f0855ecba9a..197b6861f72 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -632,6 +632,17 @@ static int msg_delete_attribute(struct ldb_module *module,
unsigned int i;
int ret;
struct ldb_message_element *el;
+ bool is_special = ldb_dn_is_special(msg->dn);
+
+ if (!is_special
+ && ltdb->cache->GUID_index_attribute != NULL
+ && ldb_attr_cmp(name, ltdb->cache->GUID_index_attribute) == 0) {
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ ldb_asprintf_errstring(ldb, "Must not modify GUID "
+ "attribute %s (used as DB index)",
+ ltdb->cache->GUID_index_attribute);
+ return LDB_ERR_CONSTRAINT_VIOLATION;
+ }
el = ldb_msg_find_element(msg, name);
if (el == NULL) {
--
2.11.0
From 751ef1aac6e33515a90ae546d77cf17b045ffb8d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:00:48 +1200
Subject: [PATCH 17/94] ldb_tdb: replace strange dn_list_cmp() in index code
This replaces dn_list_cmp() with functions that do not attempt to
to care about string termination. All index values are case sensitive
and correctly length-bound already, even for a DN index
so just use a length check and memcmp()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 27 +++++++++++++++++----------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8fd30cfd736..4db11dfe08e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -61,23 +61,28 @@ int ltdb_index_transaction_start(struct ldb_module *module)
return LDB_SUCCESS;
}
-/* compare two DN entries in a dn_list. Take account of possible
- * differences in string termination */
-static int dn_list_cmp(const struct ldb_val *v1, const struct ldb_val *v2)
+/*
+ see if two ldb_val structures contain exactly the same data
+ return -1 or 1 for a mismatch, 0 for match
+*/
+static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1,
+ const struct ldb_val *v2)
{
- if (v1->length > v2->length && v1->data[v2->length] != 0) {
+ if (v1->length > v2->length) {
return -1;
}
- if (v1->length < v2->length && v2->data[v1->length] != 0) {
+ if (v1->length < v2->length) {
return 1;
}
- return strncmp((char *)v1->data, (char *)v2->data, v1->length);
+ return memcmp(v1->data, v2->data, v1->length);
}
/*
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
- comparison with the dn returns -1 if not found
+ binary-safe comparison for the 'dn' returns -1 if not found
+
+ This is therefore safe when the value is a GUID in the future
*/
static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
const struct dn_list *list,
@@ -85,7 +90,7 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
{
unsigned int i;
for (i=0; i<list->count; i++) {
- if (dn_list_cmp(&list->dn[i], v) == 0) {
+ if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
return i;
}
}
@@ -1053,11 +1058,13 @@ static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
return;
}
- TYPESAFE_QSORT(list->dn, list->count, dn_list_cmp);
+ TYPESAFE_QSORT(list->dn, list->count,
+ ldb_val_equal_exact_for_qsort);
new_count = 1;
for (i=1; i<list->count; i++) {
- if (dn_list_cmp(&list->dn[i], &list->dn[new_count-1]) != 0) {
+ if (ldb_val_equal_exact(&list->dn[i],
+ &list->dn[new_count-1]) == 0) {
if (new_count != i) {
list->dn[new_count] = list->dn[i];
}
--
2.11.0
From b73879603d832a39a9a4a18923b16d023537eae9 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:04:43 +1200
Subject: [PATCH 18/94] ldb_tdb: Move constants into ldb_tdb.h
This helps ensure we keep these all in sync.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++--
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 197b6861f72..3c606d61820 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -145,11 +145,12 @@ bool ltdb_key_is_record(TDB_DATA key)
return true;
}
- if (key.dsize < 6) {
+ if (key.dsize < sizeof(LTDB_GUID_KEY_PREFIX)) {
return false;
}
- if (memcmp(key.dptr, "GUID=", 5) == 0) {
+ if (memcmp(key.dptr, LTDB_GUID_KEY_PREFIX,
+ sizeof(LTDB_GUID_KEY_PREFIX) - 1) == 0) {
return true;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index aa8d162fdec..e5e1e192d9f 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -71,6 +71,10 @@ struct ltdb_context {
#define LTDB_MOD_TIMESTAMP "whenChanged"
#define LTDB_OBJECTCLASS "objectClass"
+/* DB keys */
+#define LTDB_GUID_KEY_PREFIX "GUID="
+#define LTDB_GUID_SIZE 16
+
/* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c */
int ltdb_cache_reload(struct ldb_module *module);
--
2.11.0
From 76d1e3d0f4899bfc36bf8de675d582ee25df2e51 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:02:14 +1200
Subject: [PATCH 19/94] ldb_tdb: Store GUID index values in one packed ldb
attribute
This should make them more memory efficient
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 39 +++++++++++++++++++++++++++++++++++++--
1 file changed, 37 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 4db11dfe08e..3ce1d3372cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -264,8 +264,43 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
talloc_free(msg);
return ldb_module_oom(module);
}
- el->values = list->dn;
- el->num_values = list->count;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ el->values = list->dn;
+ el->num_values = list->count;
+ } else {
+ struct ldb_val v;
+ unsigned int i;
+ el->values = talloc_array(msg,
+ struct ldb_val, 1);
+ if (el->values == NULL) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+
+ v.data = talloc_array_size(el->values,
+ list->count,
+ LTDB_GUID_SIZE);
+ if (v.data == NULL) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+
+ v.length = talloc_get_size(v.data);
+
+ for (i = 0; i < list->count; i++) {
+ if (list->dn[i].length !=
+ LTDB_GUID_SIZE) {
+ talloc_free(msg);
+ return ldb_module_operr(module);
+ }
+ memcpy(&v.data[LTDB_GUID_SIZE*i],
+ list->dn[i].data,
+ LTDB_GUID_SIZE);
+ }
+ el->values[0] = v;
+ el->num_values = 1;
+ }
}
ret = ltdb_store(module, msg, TDB_REPLACE);
--
2.11.0
From 2870d8a5fe93b00f44658f063315f6a9c564dd89 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 17:26:35 +1200
Subject: [PATCH 20/94] ldb_tdb: Read GUID index values as one packed LDB
attribute
This packing should be more efficient to read than the ldb_pack format.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 27 ++++++++++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3ce1d3372cc..f2d619ddfc4 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -215,9 +215,30 @@ normal_index:
* asked for the memory to be allocated on msg, not on each
* value with LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC above
*/
- talloc_steal(el->values, msg);
- list->dn = talloc_steal(list, el->values);
- list->count = el->num_values;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ talloc_steal(el->values, msg);
+ list->dn = talloc_steal(list, el->values);
+ list->count = el->num_values;
+ } else {
+ unsigned int i;
+ static const size_t GUID_val_size = 16;
+ if (el->num_values != 1) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ list->count = el->values[0].length / GUID_val_size;
+ list->dn = talloc_array(list, struct ldb_val, list->count);
+
+ /*
+ * The actual data is on msg, due to
+ * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
+ */
+ talloc_steal(list->dn, msg);
+ for (i = 0; i < list->count; i++) {
+ list->dn[i].data = &el->values[0].data[i * GUID_val_size];
+ list->dn[i].length = GUID_val_size;
+ }
+ }
/* We don't need msg->elements any more */
talloc_free(msg->elements);
--
2.11.0
From a50766aa868fddcc05c5414d3f84425c88830297 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 15:33:14 +1200
Subject: [PATCH 21/94] ldb_tdb: Refuse to load a GUID index that is not a
multiple of 16 bytes
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f2d619ddfc4..eca5c20498e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -226,6 +226,10 @@ normal_index:
return LDB_ERR_OPERATIONS_ERROR;
}
+ if ((el->values[0].length % GUID_val_size) != 0) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
list->count = el->values[0].length / GUID_val_size;
list->dn = talloc_array(list, struct ldb_val, list->count);
--
2.11.0
From 2b8a797e132bf6d407255035a5de397eea1ba518 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:04:23 +1200
Subject: [PATCH 22/94] ldb_tdb: Write GUID index values as version 3
Nothing reads these currently, but we should refuse to load a mixed up index
in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index eca5c20498e..1dfb368b364 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -49,6 +49,8 @@ struct ltdb_idxptr {
*/
#define LTDB_INDEXING_VERSION 2
+#define LTDB_GUID_INDEXING_VERSION 3
+
/* enable the idxptr mode when transactions start */
int ltdb_index_transaction_start(struct ldb_module *module)
{
@@ -274,10 +276,20 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
return ldb_module_oom(module);
}
- ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u", LTDB_INDEXING_VERSION);
- if (ret != LDB_SUCCESS) {
- talloc_free(msg);
- return ldb_module_oom(module);
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
+ LTDB_INDEXING_VERSION);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+ } else {
+ ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
+ LTDB_GUID_INDEXING_VERSION);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
}
msg->dn = dn;
--
2.11.0
From a5fb41e0cef7f8ff54e870d5fc48457ae3618573 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 09:08:34 +1200
Subject: [PATCH 23/94] ldb_tdb: modify ltdb_delete_noindex() to take a struct
ldb_message
This will make it easier to delete records with the GUID TDB key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 16 +++++++++-------
lib/ldb/ldb_tdb/ldb_tdb.c | 7 ++++---
lib/ldb/ldb_tdb/ldb_tdb.h | 3 ++-
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 1dfb368b364..b5d2ca75031 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -263,19 +263,22 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
struct ldb_message *msg;
int ret;
+ msg = ldb_msg_new(module);
+ if (!msg) {
+ return ldb_module_oom(module);
+ }
+
+ msg->dn = dn;
+
if (list->count == 0) {
- ret = ltdb_delete_noindex(module, dn);
+ ret = ltdb_delete_noindex(module, msg);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ talloc_free(msg);
return LDB_SUCCESS;
}
return ret;
}
- msg = ldb_msg_new(module);
- if (!msg) {
- return ldb_module_oom(module);
- }
-
if (ltdb->cache->GUID_index_attribute == NULL) {
ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
LTDB_INDEXING_VERSION);
@@ -292,7 +295,6 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
}
}
- msg->dn = dn;
if (list->count > 0) {
struct ldb_message_element *el;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 3c606d61820..e13b88ff122 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -476,14 +476,15 @@ static int ltdb_add(struct ltdb_context *ctx)
delete a record from the database, not updating indexes (used for deleting
index records)
*/
-int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_delete_noindex(struct ldb_module *module,
+ const struct ldb_message *msg)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA tdb_key;
int ret;
- tdb_key = ltdb_key_dn(module, dn);
+ tdb_key = ltdb_key_dn(module, msg->dn);
if (!tdb_key.dptr) {
return LDB_ERR_OTHER;
}
@@ -516,7 +517,7 @@ static int ltdb_delete_internal(struct ldb_module *module, struct ldb_dn *dn)
goto done;
}
- ret = ltdb_delete_noindex(module, dn);
+ ret = ltdb_delete_noindex(module, msg);
if (ret != LDB_SUCCESS) {
goto done;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index e5e1e192d9f..f28936f247d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -136,7 +136,8 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
-int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
+int ltdb_delete_noindex(struct ldb_module *module,
+ const struct ldb_message *msg);
int ltdb_err_map(enum TDB_ERROR tdb_code);
struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
--
2.11.0
From c3ae15675d341680079b16a35becfc47c3993332 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:33:15 +1200
Subject: [PATCH 24/94] ldb_tdb: Delete a successful tdb_store on index add
fail in ltdb_add_internal()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index e13b88ff122..bc81612cb56 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -436,6 +436,17 @@ static int ltdb_add_internal(struct ldb_module *module,
ret = ltdb_index_add_new(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
+ /*
+ * If we failed to index, delete the message again.
+ *
+ * This is particularly important for the GUID index
+ * case, which will only fail for a duplicate DN
+ * in the index add.
+ *
+ * Note that the caller may not cancel the transation
+ * and this means the above add might really show up!
+ */
+ ltdb_delete_noindex(module, msg);
return ret;
}
--
2.11.0
From a050c874157897be8bb52aa709e98928413448cf Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 13:08:42 +1200
Subject: [PATCH 25/94] ldb_tdb: Pass the full ldb_message to ldb index
funtions
This allows the objectGUID, rather than the DN, to be the index key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 88 ++++++++++++++++++++++-----------------------
lib/ldb/ldb_tdb/ldb_tdb.c | 10 +++---
lib/ldb/ldb_tdb/ldb_tdb.h | 6 ++--
3 files changed, 52 insertions(+), 52 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b5d2ca75031..c26de922f0a 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -103,13 +103,14 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
find a entry in a dn_list. Uses a case sensitive comparison with the dn
returns -1 if not found
*/
-static int ltdb_dn_list_find_str(struct ltdb_private *ltdb,
+static int ltdb_dn_list_find_msg(struct ltdb_private *ltdb,
struct dn_list *list,
- const char *dn)
+ const struct ldb_message *msg)
{
struct ldb_val v;
- v.data = discard_const_p(unsigned char, dn);
- v.length = strlen(dn);
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ v.data = discard_const_p(unsigned char, dn_str);
+ v.length = strlen(dn_str);
return ltdb_dn_list_find_val(ltdb, list, &v);
}
@@ -1243,7 +1244,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
*/
static int ltdb_index_add1(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, int v_idx)
{
struct ldb_context *ldb;
@@ -1252,6 +1253,7 @@ static int ltdb_index_add1(struct ldb_module *module,
const struct ldb_schema_attribute *a;
struct dn_list *list;
unsigned alloc_len;
+ const char *dn_str;
ldb = ldb_module_get_ctx(module);
@@ -1283,13 +1285,14 @@ static int ltdb_index_add1(struct ldb_module *module,
ldb_debug(ldb, LDB_DEBUG_WARNING,
__location__ ": unique index violation on %s in %s, "
"conficts with %*.*s in %s",
- el->name, dn,
+ el->name, ldb_dn_get_linearized(msg->dn),
(int)list->dn[0].length,
(int)list->dn[0].length,
list->dn[0].data,
ldb_dn_get_linearized(dn_key));
ldb_asprintf_errstring(ldb, __location__ ": unique index violation on %s in %s",
- el->name, dn);
+ el->name,
+ ldb_dn_get_linearized(msg->dn));
talloc_free(list);
return LDB_ERR_ENTRY_ALREADY_EXISTS;
}
@@ -1303,13 +1306,14 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
+ dn_str = ldb_dn_get_linearized(msg->dn);
list->dn[list->count].data
- = (uint8_t *)talloc_strdup(list->dn, dn);
+ = (uint8_t *)talloc_strdup(list->dn, dn_str);
if (list->dn[list->count].data == NULL) {
talloc_free(list);
return LDB_ERR_OPERATIONS_ERROR;
}
- list->dn[list->count].length = strlen(dn);
+ list->dn[list->count].length = strlen(dn_str);
list->count++;
ret = ltdb_dn_list_store(module, dn_key, list);
@@ -1324,13 +1328,13 @@ static int ltdb_index_add1(struct ldb_module *module,
*/
static int ltdb_index_add_el(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
unsigned int i;
for (i = 0; i < el->num_values; i++) {
int ret = ltdb_index_add1(module, ltdb,
- dn, el, i);
+ msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1344,14 +1348,19 @@ static int ltdb_index_add_el(struct ldb_module *module,
*/
static int ltdb_index_add_all(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
- struct ldb_message_element *elements,
- unsigned int num_el)
+ const struct ldb_message *msg)
{
+ struct ldb_message_element *elements = msg->elements;
unsigned int i;
+ const char *dn_str;
- if (dn[0] == '@') {
- return LDB_SUCCESS;
+ if (ldb_dn_is_special(msg->dn)) {
+ return LDB_SUCCESS;
+ }
+
+ dn_str = ldb_dn_get_linearized(msg->dn);
+ if (dn_str == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
}
if (!ltdb->cache->attribute_indexes) {
@@ -1359,17 +1368,19 @@ static int ltdb_index_add_all(struct ldb_module *module,
return LDB_SUCCESS;
}
- for (i = 0; i < num_el; i++) {
+ for (i = 0; i < msg->num_elements; i++) {
int ret;
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
- ret = ltdb_index_add_el(module, ltdb, dn, &elements[i]);
+ ret = ltdb_index_add_el(module, ltdb,
+ msg, &elements[i]);
if (ret != LDB_SUCCESS) {
struct ldb_context *ldb = ldb_module_get_ctx(module);
ldb_asprintf_errstring(ldb,
__location__ ": Failed to re-index %s in %s - %s",
- elements[i].name, dn, ldb_errstring(ldb));
+ elements[i].name, dn_str,
+ ldb_errstring(ldb));
return ret;
}
}
@@ -1420,9 +1431,9 @@ static int ltdb_index_onelevel(struct ldb_module *module,
el.num_values = 1;
if (add) {
- ret = ltdb_index_add1(module, ltdb, dn, &el, 0);
+ ret = ltdb_index_add1(module, ltdb, msg, &el, 0);
} else { /* delete */
- ret = ltdb_index_del_value(module, ltdb, msg->dn, &el, 0);
+ ret = ltdb_index_del_value(module, ltdb, msg, &el, 0);
}
talloc_free(pdn);
@@ -1436,17 +1447,16 @@ static int ltdb_index_onelevel(struct ldb_module *module,
*/
int ltdb_index_add_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
- if (ldb_dn_is_special(dn)) {
+ if (ldb_dn_is_special(msg->dn)) {
return LDB_SUCCESS;
}
if (!ltdb_is_indexed(module, ltdb, el->name)) {
return LDB_SUCCESS;
}
- return ltdb_index_add_el(module, ltdb,
- ldb_dn_get_linearized(dn), el);
+ return ltdb_index_add_el(module, ltdb, msg, el);
}
/*
@@ -1456,20 +1466,13 @@ int ltdb_index_add_new(struct ldb_module *module,
struct ltdb_private *ltdb,
const struct ldb_message *msg)
{
- const char *dn;
int ret;
if (ldb_dn_is_special(msg->dn)) {
return LDB_SUCCESS;
}
- dn = ldb_dn_get_linearized(msg->dn);
- if (dn == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ret = ltdb_index_add_all(module, ltdb, dn, msg->elements,
- msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1483,7 +1486,7 @@ int ltdb_index_add_new(struct ldb_module *module,
*/
int ltdb_index_del_value(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, unsigned int v_idx)
{
struct ldb_context *ldb;
@@ -1492,6 +1495,7 @@ int ltdb_index_del_value(struct ldb_module *module,
int ret, i;
unsigned int j;
struct dn_list *list;
+ struct ldb_dn *dn = msg->dn;
ldb = ldb_module_get_ctx(module);
@@ -1528,7 +1532,7 @@ int ltdb_index_del_value(struct ldb_module *module,
return ret;
}
- i = ltdb_dn_list_find_str(ltdb, list, dn_str);
+ i = ltdb_dn_list_find_msg(ltdb, list, msg);
if (i == -1) {
/* nothing to delete */
talloc_free(dn_key);
@@ -1560,7 +1564,7 @@ int ltdb_index_del_value(struct ldb_module *module,
*/
int ltdb_index_del_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
const char *dn_str;
@@ -1572,7 +1576,7 @@ int ltdb_index_del_element(struct ldb_module *module,
return LDB_SUCCESS;
}
- dn_str = ldb_dn_get_linearized(dn);
+ dn_str = ldb_dn_get_linearized(msg->dn);
if (dn_str == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1585,7 +1589,7 @@ int ltdb_index_del_element(struct ldb_module *module,
return LDB_SUCCESS;
}
for (i = 0; i < el->num_values; i++) {
- ret = ltdb_index_del_value(module, ltdb, dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1620,7 +1624,7 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
for (i = 0; i < msg->num_elements; i++) {
ret = ltdb_index_del_element(module, ltdb,
- msg->dn, &msg->elements[i]);
+ msg, &msg->elements[i]);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1789,7 +1793,6 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
struct ltdb_private);
struct ldb_message *msg;
- const char *dn = NULL;
unsigned int nb_elements_in_db;
const struct ldb_val val = {
.data = data.dptr,
@@ -1836,8 +1839,6 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
(char *)key.dptr);
talloc_free(msg);
return -1;
- } else {
- dn = ldb_dn_get_linearized(msg->dn);
}
ret = ltdb_index_onelevel(module, msg, 1);
@@ -1849,8 +1850,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
return -1;
}
- ret = ltdb_index_add_all(module, ltdb, dn,
- msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
ctx->error = ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index bc81612cb56..1d00574fc8a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -663,7 +663,7 @@ static int msg_delete_attribute(struct ldb_module *module,
}
i = el - msg->elements;
- ret = ltdb_index_del_element(module, ltdb, msg->dn, el);
+ ret = ltdb_index_del_element(module, ltdb, msg, el);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -722,7 +722,7 @@ static int msg_delete_element(struct ldb_module *module,
ltdb, msg, name);
}
- ret = ltdb_index_del_value(module, ltdb, msg->dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -847,7 +847,7 @@ int ltdb_modify_internal(struct ldb_module *module,
goto done;
}
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn,
+ msg2,
el);
if (ret != LDB_SUCCESS) {
goto done;
@@ -929,7 +929,7 @@ int ltdb_modify_internal(struct ldb_module *module,
el2->num_values += el->num_values;
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn, el);
+ msg2, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -1008,7 +1008,7 @@ int ltdb_modify_internal(struct ldb_module *module,
}
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn, el);
+ msg2, el);
if (ret != LDB_SUCCESS) {
goto done;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f28936f247d..d6b53e8d801 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -93,15 +93,15 @@ int ltdb_index_add_new(struct ldb_module *module,
int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_del_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el);
int ltdb_index_add_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el);
int ltdb_index_del_value(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, unsigned int v_idx);
int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
--
2.11.0
From 4069048a8ffd56f40e1ef2ee1e1c9db87377be10 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:13:32 +1200
Subject: [PATCH 26/94] ldb_tdb: Optionally use a GUID index key in
ltdb_dn_list_find_msg()
This function is used to find an existing index value and this
change allows it to find the value by GUID rather than by DN once
the GUID index is enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index c26de922f0a..184c4b49e97 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -108,9 +108,19 @@ static int ltdb_dn_list_find_msg(struct ltdb_private *ltdb,
const struct ldb_message *msg)
{
struct ldb_val v;
- const char *dn_str = ldb_dn_get_linearized(msg->dn);
- v.data = discard_const_p(unsigned char, dn_str);
- v.length = strlen(dn_str);
+ const struct ldb_val *key_val;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ v.data = discard_const_p(unsigned char, dn_str);
+ v.length = strlen(dn_str);
+ } else {
+ key_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (key_val == NULL) {
+ return -1;
+ }
+ v = *key_val;
+ }
return ltdb_dn_list_find_val(ltdb, list, &v);
}
--
2.11.0
From 27d15f049efe1d36a3d302005b2b9b25f584e710 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:18:19 +1200
Subject: [PATCH 27/94] ldb_tdb: Optionally store a GUID as the index record
This allows, when enabled, the index record to contain (say) the objectGUID, not the DN
of the record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 35 +++++++++++++++++++++++++++--------
1 file changed, 27 insertions(+), 8 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 184c4b49e97..d799eaf90f9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1263,7 +1263,6 @@ static int ltdb_index_add1(struct ldb_module *module,
const struct ldb_schema_attribute *a;
struct dn_list *list;
unsigned alloc_len;
- const char *dn_str;
ldb = ldb_module_get_ctx(module);
@@ -1316,14 +1315,34 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn_str = ldb_dn_get_linearized(msg->dn);
- list->dn[list->count].data
- = (uint8_t *)talloc_strdup(list->dn, dn_str);
- if (list->dn[list->count].data == NULL) {
- talloc_free(list);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ list->dn[list->count].data
+ = (uint8_t *)talloc_strdup(list->dn, dn_str);
+ if (list->dn[list->count].data == NULL) {
+ talloc_free(list);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->dn[list->count].length = strlen(dn_str);
+ } else {
+ const struct ldb_val *key_val;
+ key_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (key_val == NULL) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
+
+ if (key_val->length != LTDB_GUID_SIZE) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
+ list->dn[list->count] = ldb_val_dup(list->dn, key_val);
+ if (list->dn[list->count].data == NULL) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
}
- list->dn[list->count].length = strlen(dn_str);
list->count++;
ret = ltdb_dn_list_store(module, dn_key, list);
--
2.11.0
From 39cb3a5bfa1275e06cbb0c69c82ba7895bad31d9 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:39:08 +1200
Subject: [PATCH 28/94] ldb_tdb: Implement ltdb_search_base() for a GUID index
The GUID index case can not directly use ltdb_key_dn() and tdb_exists() to
show that a records exists.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 32 +++++++++++++++++++++++---------
1 file changed, 23 insertions(+), 9 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 753eb940048..661d9704422 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -115,24 +115,38 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
*/
static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
{
- void *data = ldb_module_get_private(module);
- struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- TDB_DATA tdb_key;
int exists;
+ int ret;
+ struct ldb_message *msg = NULL;
if (ldb_dn_is_null(dn)) {
return LDB_ERR_NO_SUCH_OBJECT;
}
- /* form the key */
- tdb_key = ltdb_key_dn(module, dn);
- if (!tdb_key.dptr) {
+ /*
+ * We can't use tdb_exists() directly on a key when the TDB
+ * key is the GUID one, not the DN based one. So we just do a
+ * normal search and avoid most of the allocation with the
+ * LDB_UNPACK_DATA_FLAG_NO_DN and
+ * LDB_UNPACK_DATA_FLAG_NO_ATTRS flags
+ */
+ msg = ldb_msg_new(module);
+ if (msg == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
- exists = tdb_exists(ltdb->tdb, tdb_key);
- talloc_free(tdb_key.dptr);
-
+ ret = ltdb_search_dn1(module, dn,
+ msg,
+ LDB_UNPACK_DATA_FLAG_NO_DN|
+ LDB_UNPACK_DATA_FLAG_NO_ATTRS);
+ talloc_free(msg);
+ if (ret == LDB_SUCCESS) {
+ exists = true;
+ } else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ exists = false;
+ } else {
+ return ret;
+ }
if (exists) {
return LDB_SUCCESS;
}
--
2.11.0
From 5e38a7183b3428808b8cb1c88aaf59985a2dfee9 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 11:33:17 +1200
Subject: [PATCH 29/94] ldb_tdb: Prepare to handle rename with GUID index by
using ltdb_search_base()
This will allow use of a GUID TDB key in the future. While ltdb_search_base()
might be marginally slower than tdb_exists(), no allocation is done for the
attributes or DN, and renmaes are not a very common operation.
This allows a check if the target DN exists even when the direct DN -> key
link is broken.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 2 +-
lib/ldb/ldb_tdb/ldb_tdb.c | 44 ++++++++++++++++++++++++++++++++------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
3 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 661d9704422..555e7cb0b66 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -113,7 +113,7 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
{
int exists;
int ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1d00574fc8a..7dd14202d38 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1145,7 +1145,11 @@ static int ltdb_rename(struct ltdb_context *ctx)
/* We need to, before changing the DB, check if the new DN
* exists, so we can return this error to the caller with an
- * unmodified DB */
+ * unmodified DB
+ *
+ * Even in GUID index mode we use ltdb_key_dn() as we are
+ * trying to figure out if this is just a case rename
+ */
tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
@@ -1159,19 +1163,35 @@ static int ltdb_rename(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
- /* Only declare a conflict if the new DN already exists, and it isn't a case change on the old DN */
- if (tdb_key_old.dsize != tdb_key.dsize || memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
- if (tdb_exists(ltdb->tdb, tdb_key)) {
- talloc_free(tdb_key_old.dptr);
- talloc_free(tdb_key.dptr);
- ldb_asprintf_errstring(ldb_module_get_ctx(module),
- "Entry %s already exists",
- ldb_dn_get_linearized(req->op.rename.newdn));
- /* finding the new record already in the DB is an error */
- talloc_free(msg);
- return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ /*
+ * Only declare a conflict if the new DN already exists,
+ * and it isn't a case change on the old DN
+ */
+ if (tdb_key_old.dsize != tdb_key.dsize
+ || memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
+ ret = ltdb_search_base(module,
+ req->op.rename.newdn);
+ if (ret == LDB_SUCCESS) {
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ } else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ ret = LDB_SUCCESS;
}
}
+
+ /* finding the new record already in the DB is an error */
+
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Entry %s already exists",
+ ldb_dn_get_linearized(req->op.rename.newdn));
+ }
+ if (ret != LDB_SUCCESS) {
+ talloc_free(tdb_key_old.dptr);
+ talloc_free(tdb_key.dptr);
+ talloc_free(msg);
+ return ret;
+ }
+
talloc_free(tdb_key_old.dptr);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index d6b53e8d801..e9d74664922 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -115,6 +115,7 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
+int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn);
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
struct TDB_DATA tdb_key,
struct ldb_message *msg,
--
2.11.0
From dc3a5f0a085742525e4e67a4381675ed5a2e0451 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 12:58:58 +1200
Subject: [PATCH 30/94] ldb_tdb: Split ltdb_index_onelevel() into a helper
function
This will allow the code to be re-used for storing the DN->GUID index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 74 ++++++++++++++++++++++++++++-----------------
1 file changed, 47 insertions(+), 27 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index d799eaf90f9..f41ce258ba4 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1419,43 +1419,25 @@ static int ltdb_index_add_all(struct ldb_module *module,
/*
- insert a one level index for a message
+ insert a DN index for a message
*/
-static int ltdb_index_onelevel(struct ldb_module *module,
- const struct ldb_message *msg, int add)
-{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
- struct ltdb_private);
+static int ltdb_modify_index_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg,
+ struct ldb_dn *dn,
+ const char *index, int add)
+{
struct ldb_message_element el;
struct ldb_val val;
- struct ldb_dn *pdn;
- const char *dn;
int ret;
- /* We index for ONE Level only if requested */
- if (!ltdb->cache->one_level_indexes) {
- return LDB_SUCCESS;
- }
-
- pdn = ldb_dn_get_parent(module, msg->dn);
- if (pdn == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- dn = ldb_dn_get_linearized(msg->dn);
- if (dn == NULL) {
- talloc_free(pdn);
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(pdn));
+ val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
if (val.data == NULL) {
- talloc_free(pdn);
return LDB_ERR_OPERATIONS_ERROR;
}
val.length = strlen((char *)val.data);
- el.name = LTDB_IDXONE;
+ el.name = index;
el.values = &val;
el.num_values = 1;
@@ -1465,6 +1447,44 @@ static int ltdb_index_onelevel(struct ldb_module *module,
ret = ltdb_index_del_value(module, ltdb, msg, &el, 0);
}
+ if (ret != LDB_SUCCESS) {
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb,
+ __location__
+ ": Failed to modify %s "
+ "against %s in %s - %s",
+ index,
+ ltdb->cache->GUID_index_attribute,
+ dn_str, ldb_errstring(ldb));
+ return ret;
+ }
+ return ret;
+}
+
+/*
+ insert a one level index for a message
+*/
+static int ltdb_index_onelevel(struct ldb_module *module,
+ const struct ldb_message *msg, int add)
+{
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
+ struct ldb_dn *pdn;
+ int ret;
+
+ /* We index for ONE Level only if requested */
+ if (!ltdb->cache->one_level_indexes) {
+ return LDB_SUCCESS;
+ }
+
+ pdn = ldb_dn_get_parent(module, msg->dn);
+ if (pdn == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ret = ltdb_modify_index_dn(module, ltdb,
+ msg, pdn, LTDB_IDXONE, add);
+
talloc_free(pdn);
return ret;
--
2.11.0
From 970bd997a1b403d0b918fc8164646ab78208698a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 12:25:42 +1200
Subject: [PATCH 31/94] ldb_tdb: Add/remove a GUID index of the DN during
ltdb_index_add_all()/ltdb_index_delete()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 41 +++++++++++++++++++++++++++++++++++++----
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
2 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f41ce258ba4..2a95af95086 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -44,6 +44,10 @@ struct ltdb_idxptr {
int error;
};
+static int ltdb_write_index_dn_guid(struct ldb_module *module,
+ const struct ldb_message *msg,
+ int add);
+
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
*/
@@ -1382,23 +1386,28 @@ static int ltdb_index_add_all(struct ldb_module *module,
struct ldb_message_element *elements = msg->elements;
unsigned int i;
const char *dn_str;
+ int ret;
- if (ldb_dn_is_special(msg->dn)) {
- return LDB_SUCCESS;
- }
+ if (ldb_dn_is_special(msg->dn)) {
+ return LDB_SUCCESS;
+ }
dn_str = ldb_dn_get_linearized(msg->dn);
if (dn_str == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
+ ret = ltdb_write_index_dn_guid(module, msg, 1);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if (!ltdb->cache->attribute_indexes) {
/* no indexed fields */
return LDB_SUCCESS;
}
for (i = 0; i < msg->num_elements; i++) {
- int ret;
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
@@ -1491,6 +1500,25 @@ static int ltdb_index_onelevel(struct ldb_module *module,
}
/*
+ insert a one level index for a message
+*/
+static int ltdb_write_index_dn_guid(struct ldb_module *module,
+ const struct ldb_message *msg,
+ int add)
+{
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
+
+ /* We index for DN only if using a GUID index */
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ return LDB_SUCCESS;
+ }
+
+ return ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
+ LTDB_IDXDN, add);
+}
+
+/*
add the index entries for a new element in a record
The caller guarantees that these element values are not yet indexed
*/
@@ -1666,6 +1694,11 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
return ret;
}
+ ret = ltdb_write_index_dn_guid(module, msg, 0);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if (!ltdb->cache->attribute_indexes) {
/* no indexed fields */
return LDB_SUCCESS;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index e9d74664922..75759c697a2 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -60,6 +60,7 @@ struct ltdb_context {
#define LTDB_IDXVERSION "@IDXVERSION"
#define LTDB_IDXATTR "@IDXATTR"
#define LTDB_IDXONE "@IDXONE"
+#define LTDB_IDXDN "@IDXDN"
#define LTDB_BASEINFO "@BASEINFO"
#define LTDB_OPTIONS "@OPTIONS"
#define LTDB_ATTRIBUTES "@ATTRIBUTES"
--
2.11.0
From 3c95269f1facd687c87a428e5c9bd78e65f47ec5 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:21:00 +1200
Subject: [PATCH 32/94] ldb_tdb: Give LDB_ERR_ENTRY_ALREADY_EXISTS when a
duplicate DN index is detected
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 2a95af95086..e7f71dc79d5 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1506,6 +1506,7 @@ static int ltdb_write_index_dn_guid(struct ldb_module *module,
const struct ldb_message *msg,
int add)
{
+ int ret;
struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
struct ltdb_private);
@@ -1514,8 +1515,16 @@ static int ltdb_write_index_dn_guid(struct ldb_module *module,
return LDB_SUCCESS;
}
- return ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
- LTDB_IDXDN, add);
+ ret = ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
+ LTDB_IDXDN, add);
+
+ if (ret == LDB_ERR_CONSTRAINT_VIOLATION) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Entry %s already exists",
+ ldb_dn_get_linearized(msg->dn));
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ }
+ return ret;
}
/*
--
2.11.0
From 04a9119e16e7b8f5e96b77c51c5cabe096afa832 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:21:26 +1200
Subject: [PATCH 33/94] ldb_tdb: Give LDB_ERR_CONSTRAINT_VIOLATION when a
duplicate GUID index is detected
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 7dd14202d38..c86faa5d6fd 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -328,7 +328,18 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
ret = tdb_store(ltdb->tdb, tdb_key, tdb_data, flgs);
if (ret != 0) {
+ bool is_special = ldb_dn_is_special(msg->dn);
ret = ltdb_err_map(tdb_error(ltdb->tdb));
+
+ /*
+ * LDB_ERR_ENTRY_ALREADY_EXISTS means the DN, not
+ * the GUID, so re-map
+ */
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS
+ && !is_special
+ && ltdb->cache->GUID_index_attribute != NULL) {
+ ret = LDB_ERR_CONSTRAINT_VIOLATION;
+ }
goto done;
}
--
2.11.0
From 6c4c69335662ccda6770e01964e36d98909c8475 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 13:26:30 +1200
Subject: [PATCH 34/94] ldb_tdb: Pass ltdb_private to ltdb_dn_list_load()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index e7f71dc79d5..67ff3fe0e32 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -166,12 +166,12 @@ static struct dn_list *ltdb_index_idxptr(struct ldb_module *module, TDB_DATA rec
struct dn_list
*/
static int ltdb_dn_list_load(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_dn *dn, struct dn_list *list)
{
struct ldb_message *msg;
int ret;
struct ldb_message_element *el;
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
TDB_DATA rec;
struct dn_list *list2;
TDB_DATA key;
@@ -635,7 +635,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
dn = ltdb_index_key(ldb, tree->u.equality.attr, &tree->u.equality.value, NULL);
if (!dn) return LDB_ERR_OPERATIONS_ERROR;
- ret = ltdb_dn_list_load(module, dn, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn, list);
talloc_free(dn);
return ret;
}
@@ -979,6 +979,7 @@ static int ltdb_index_dn_and(struct ldb_module *module,
return a list of matching objects using a one-level index
*/
static int ltdb_index_dn_one(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_dn *parent_dn,
struct dn_list *list)
{
@@ -998,7 +999,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_dn_list_load(module, key, list);
+ ret = ltdb_dn_list_load(module, ltdb, key, list);
talloc_free(key);
if (ret != LDB_SUCCESS) {
return ret;
@@ -1209,7 +1210,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_dn_one(ac->module, ac->base, dn_list);
+ ret = ltdb_index_dn_one(ac->module, ltdb, ac->base, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
return ret;
@@ -1282,7 +1283,7 @@ static int ltdb_index_add1(struct ldb_module *module,
}
talloc_steal(list, dn_key);
- ret = ltdb_dn_list_load(module, dn_key, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
talloc_free(list);
return ret;
@@ -1605,7 +1606,7 @@ int ltdb_index_del_value(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_dn_list_load(module, dn_key, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* it wasn't indexed. Did we have an earlier error? If we did then
its gone now */
--
2.11.0
From 0806eac06148d8186781fc36d379b2254d9afb03 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 17:51:40 +1200
Subject: [PATCH 35/94] ldb_tdb: Do not directly assign DN into the index
result list
With the GUID index option, the values in the index result list may not be a DN
but the objectGUID. We look up the @IDXDN index to get that if required.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 91 +++++++++++++++++++++++++++++++++------------
1 file changed, 68 insertions(+), 23 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 67ff3fe0e32..b0a8f723fc0 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -651,6 +651,8 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
const struct ldb_parse_tree *tree,
struct dn_list *list)
{
+ /* Must be at the same scope as the ltdb_index_dn_simple */
+ struct ldb_parse_tree dn_tree;
if (ltdb->disallow_dn_filter &&
(ldb_attr_cmp(tree->u.equality.attr, "dn") == 0)) {
/* in AD mode we do not support "(dn=...)" search filters */
@@ -659,14 +661,21 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
return LDB_SUCCESS;
}
if (ldb_attr_dn(tree->u.equality.attr) == 0) {
- list->dn = talloc_array(list, struct ldb_val, 1);
- if (list->dn == NULL) {
- ldb_module_oom(module);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ list->dn = talloc_array(list, struct ldb_val, 1);
+ if (list->dn == NULL) {
+ ldb_module_oom(module);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->dn[0] = tree->u.equality.value;
+ list->count = 1;
+ return LDB_SUCCESS;
}
- list->dn[0] = tree->u.equality.value;
- list->count = 1;
- return LDB_SUCCESS;
+
+ /* Overwrite the attr with @IDXDN */
+ dn_tree = *tree;
+ dn_tree.u.equality.attr = LTDB_IDXDN;
+ tree = &dn_tree;
}
return ltdb_index_dn_simple(module, ltdb, tree, list);
}
@@ -978,10 +987,11 @@ static int ltdb_index_dn_and(struct ldb_module *module,
/*
return a list of matching objects using a one-level index
*/
-static int ltdb_index_dn_one(struct ldb_module *module,
- struct ltdb_private *ltdb,
- struct ldb_dn *parent_dn,
- struct dn_list *list)
+static int ltdb_index_dn_attr(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *attr,
+ struct ldb_dn *dn,
+ struct dn_list *list)
{
struct ldb_context *ldb;
struct ldb_dn *key;
@@ -991,9 +1001,9 @@ static int ltdb_index_dn_one(struct ldb_module *module,
ldb = ldb_module_get_ctx(module);
/* work out the index key from the parent DN */
- val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(parent_dn));
+ val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
val.length = strlen((char *)val.data);
- key = ltdb_index_key(ldb, LTDB_IDXONE, &val, NULL);
+ key = ltdb_index_key(ldb, attr, &val, NULL);
if (!key) {
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1013,6 +1023,47 @@ static int ltdb_index_dn_one(struct ldb_module *module,
}
/*
+ return a list of matching objects using a one-level index
+ */
+static int ltdb_index_dn_one(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *parent_dn,
+ struct dn_list *list)
+{
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXONE, parent_dn, list);
+}
+
+/*
+ return a list of matching objects using the DN index
+ */
+static int ltdb_index_dn_base_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *base_dn,
+ struct dn_list *dn_list)
+{
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXDN, base_dn, dn_list);
+ }
+
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ talloc_free(dn_list);
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = discard_const_p(unsigned char,
+ ldb_dn_get_linearized(base_dn));
+ if (dn_list->dn[0].data == NULL) {
+ talloc_free(dn_list);
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
+ dn_list->count = 1;
+ return LDB_SUCCESS;
+}
+
+/*
return a list of dn's that might match a indexed search or
an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
*/
@@ -1191,18 +1242,12 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
switch (ac->scope) {
case LDB_SCOPE_BASE:
- dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
- if (dn_list->dn == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(ac->module);
- }
- dn_list->dn[0].data = discard_const_p(unsigned char, ldb_dn_get_linearized(ac->base));
- if (dn_list->dn[0].data == NULL) {
+ ret = ltdb_index_dn_base_dn(ac->module, ltdb,
+ ac->base, dn_list);
+ if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
- return ldb_module_oom(ac->module);
+ return ret;
}
- dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
- dn_list->count = 1;
break;
case LDB_SCOPE_ONELEVEL:
--
2.11.0
From 7066a6f2978f1938cd1451ab524ffc2804ee9daa Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:33:24 +1200
Subject: [PATCH 36/94] ldb_tdb: add control points for the new GUID index mode
The @IDXGUID attribute in the @INDEXLIST will be objectGUID
in Samba.
The @IDX_DN_GUID attribute in the @INDEXLIST will be GUID
in Samba.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 75759c697a2..4ef4271df9a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -61,6 +61,8 @@ struct ltdb_context {
#define LTDB_IDXATTR "@IDXATTR"
#define LTDB_IDXONE "@IDXONE"
#define LTDB_IDXDN "@IDXDN"
+#define LTDB_IDXGUID "@IDXGUID"
+#define LTDB_IDX_DN_GUID "@IDX_DN_GUID"
#define LTDB_BASEINFO "@BASEINFO"
#define LTDB_OPTIONS "@OPTIONS"
#define LTDB_ATTRIBUTES "@ATTRIBUTES"
--
2.11.0
From 6a3e4a5a36238e64a5f178598b96884d6b452e97 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 10:42:40 +1200
Subject: [PATCH 37/94] ldb_tdb: Load the syntax of the GUID index attr during
ltdb_cache_load()
This allows us to use the ldif_write function later to create a string GUID for the TDB key.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 12 ++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 2 ++
2 files changed, 14 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index f08e073af1a..137ddf97fe8 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -371,6 +371,7 @@ int ltdb_cache_load(struct ldb_module *module)
struct ldb_dn *baseinfo_dn = NULL, *options_dn = NULL;
uint64_t seq;
struct ldb_message *baseinfo = NULL, *options = NULL;
+ const struct ldb_schema_attribute *a;
int r;
ldb = ldb_module_get_ctx(module);
@@ -474,6 +475,17 @@ int ltdb_cache_load(struct ldb_module *module)
goto failed;
}
+ ltdb->GUID_index_syntax = NULL;
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ /*
+ * Now the attributes are loaded, set the guid_index_syntax.
+ * This can't fail, it will return a default at worst
+ */
+ a = ldb_schema_attribute_by_name(ldb,
+ ltdb->cache->GUID_index_attribute);
+ ltdb->GUID_index_syntax = a->syntax;
+ }
+
done:
talloc_free(options);
talloc_free(baseinfo);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 4ef4271df9a..db73632bf93 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,8 @@ struct ltdb_private {
bool warn_unindexed;
bool warn_reindex;
+
+ const struct ldb_schema_syntax *GUID_index_syntax;
};
struct ltdb_context {
--
2.11.0
From c771d175e050650eecd1b68e3ddc6970f5d0ffd3 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 10:44:34 +1200
Subject: [PATCH 38/94] ldb_tdb: Add a function to take a GUID and make the
TDB_DATA key
This allows us to format the TDB key as DN=GUID=f7c953ee-cf9c-433f-b423-21ce04d09591
and so be compatible with an un-indexed search and a re-index with an old ldb.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 26 ++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 30 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index c86faa5d6fd..6392201694e 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -210,6 +210,32 @@ failed:
return key;
}
+TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *GUID_val)
+{
+ TDB_DATA key;
+ const char *GUID_prefix = "GUID=";
+ const int GUID_prefix_len = strlen(GUID_prefix);
+
+ key.dptr = talloc_size(mem_ctx,
+ GUID_val->length+GUID_prefix_len);
+
+ if (key.dptr == NULL) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ memcpy(key.dptr, "GUID=", GUID_prefix_len);
+ memcpy(&key.dptr[GUID_prefix_len],
+ GUID_val->data, GUID_val->length);
+
+ key.dsize = talloc_get_size(key.dptr);
+ return key;
+}
+
/*
form a TDB_DATA for a record key
caller frees
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index db73632bf93..5aa0d5f35fb 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -140,6 +140,10 @@ int ltdb_unlock_read(struct ldb_module *module);
bool ltdb_key_is_record(TDB_DATA key);
TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
+TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *guid_val);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From 149dc963faafe61da183d6010fdd0806272f20f3 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 18:09:01 +1200
Subject: [PATCH 39/94] ldb_tdb: Add a function to get the GUID key for a DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 40 ++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 5 +++++
2 files changed, 45 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b0a8f723fc0..272b622db0c 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -47,6 +47,10 @@ struct ltdb_idxptr {
static int ltdb_write_index_dn_guid(struct ldb_module *module,
const struct ldb_message *msg,
int add);
+static int ltdb_index_dn_base_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *base_dn,
+ struct dn_list *dn_list);
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
@@ -266,6 +270,42 @@ normal_index:
return LDB_SUCCESS;
}
+int ltdb_key_dn_from_idx(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ TDB_DATA *tdb_key)
+{
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ int ret;
+ struct dn_list *list = talloc(mem_ctx, struct dn_list);
+ if (list == NULL) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ltdb_index_dn_base_dn(module, ltdb, dn, list);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
+ if (list->count == 0) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ if (list->count > 1) {
+ return LDB_ERR_CONSTRAINT_VIOLATION;
+ }
+
+ *tdb_key = ltdb_guid_to_key(module, ltdb,
+ mem_ctx, &list->dn[0]);
+ if (tdb_key->dptr == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ return LDB_SUCCESS;
+}
+
+
/*
save a dn_list into a full @IDX style record
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 5aa0d5f35fb..853ae7d3b9c 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -112,6 +112,11 @@ int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
int ltdb_index_transaction_commit(struct ldb_module *module);
int ltdb_index_transaction_cancel(struct ldb_module *module);
+int ltdb_key_dn_from_idx(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ TDB_DATA *tdb_key);
/* The following definitions come from lib/ldb/ldb_tdb/ldb_search.c */
--
2.11.0
From a20476332bd0a66c3c2f4682ebf7f5bf9ebf04f4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 16:13:42 +1200
Subject: [PATCH 40/94] ldb_tdb: Add an index shortcut for a <GUID= DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 47 ++++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 272b622db0c..07a018beb0d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1082,25 +1082,42 @@ static int ltdb_index_dn_base_dn(struct ldb_module *module,
struct ldb_dn *base_dn,
struct dn_list *dn_list)
{
- if (ltdb->cache->GUID_index_attribute != NULL) {
- return ltdb_index_dn_attr(module, ltdb,
- LTDB_IDXDN, base_dn, dn_list);
+ const struct ldb_val *guid_val = NULL;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = discard_const_p(unsigned char,
+ ldb_dn_get_linearized(base_dn));
+ if (dn_list->dn[0].data == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
+ dn_list->count = 1;
+
+ return LDB_SUCCESS;
}
- dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
- if (dn_list->dn == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(module);
+ if (ltdb->cache->GUID_index_dn_component != NULL) {
+ guid_val = ldb_dn_get_extended_component(base_dn,
+ ltdb->cache->GUID_index_dn_component);
}
- dn_list->dn[0].data = discard_const_p(unsigned char,
- ldb_dn_get_linearized(base_dn));
- if (dn_list->dn[0].data == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(module);
+
+ if (guid_val != NULL) {
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = guid_val->data;
+ dn_list->dn[0].length = guid_val->length;
+ dn_list->count = 1;
+
+ return LDB_SUCCESS;
}
- dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
- dn_list->count = 1;
- return LDB_SUCCESS;
+
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXDN, base_dn, dn_list);
}
/*
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 853ae7d3b9c..1e639e9a60a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -21,6 +21,7 @@ struct ltdb_private {
bool one_level_indexes;
bool attribute_indexes;
const char *GUID_index_attribute;
+ const char *GUID_index_dn_component;
} *cache;
int in_transaction;
--
2.11.0
From 926a8180223920d6ddf048c56f00fc2b154fdd2d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 17:27:23 +1200
Subject: [PATCH 41/94] ldb_tdb: Check version number on index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 29 ++++++++++++++++++++++++++---
1 file changed, 26 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 07a018beb0d..1d49533be79 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -174,7 +174,7 @@ static int ltdb_dn_list_load(struct ldb_module *module,
struct ldb_dn *dn, struct dn_list *list)
{
struct ldb_message *msg;
- int ret;
+ int ret, version;
struct ldb_message_element *el;
TDB_DATA rec;
struct dn_list *list2;
@@ -222,14 +222,14 @@ normal_index:
return ret;
}
- /* TODO: check indexing version number */
-
el = ldb_msg_find_element(msg, LTDB_IDX);
if (!el) {
talloc_free(msg);
return LDB_SUCCESS;
}
+ version = ldb_msg_find_attr_as_int(msg, LTDB_IDXVERSION, 0);
+
/*
* we avoid copying the strings by stealing the list. We have
* to steal msg onto el->values (which looks odd) because we
@@ -237,12 +237,35 @@ normal_index:
* value with LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC above
*/
if (ltdb->cache->GUID_index_attribute == NULL) {
+ /* check indexing version number */
+ if (version != LTDB_INDEXING_VERSION) {
+ ldb_debug_set(ldb_module_get_ctx(module),
+ LDB_DEBUG_ERROR,
+ "Wrong DN index version %d "
+ "expected %d for %s",
+ version, LTDB_INDEXING_VERSION,
+ ldb_dn_get_linearized(dn));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
talloc_steal(el->values, msg);
list->dn = talloc_steal(list, el->values);
list->count = el->num_values;
} else {
unsigned int i;
static const size_t GUID_val_size = 16;
+ if (version != LTDB_GUID_INDEXING_VERSION) {
+ /* This is quite likely during the DB startup
+ on first upgrade to using a GUID index */
+ ldb_debug_set(ldb_module_get_ctx(module),
+ LDB_DEBUG_ERROR,
+ "Wrong GUID index version %d "
+ "expected %d for %s",
+ version, LTDB_GUID_INDEXING_VERSION,
+ ldb_dn_get_linearized(dn));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
if (el->num_values != 1) {
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From c77dab2efa2a66933f10cd5713695ac8bde60afb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 15:47:15 +1200
Subject: [PATCH 42/94] ldb_tdb: Add mem_ctx to ltdb_key_dn() and
ltdb_key_msg()
This follows modern Samba coding style where memory
returned is allocated on a supplied memory context.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
lib/ldb/ldb_tdb/ldb_search.c | 15 ++++++++++++---
lib/ldb/ldb_tdb/ldb_tdb.c | 43 ++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.h | 6 ++++--
4 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 1d49533be79..f6dfb8fc3c9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1955,7 +1955,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
/* check if the DN key has changed, perhaps due to the case
insensitivity of an element changing, or a change from DN
to GUID keys */
- key2 = ltdb_key_msg(module, msg);
+ key2 = ltdb_key_msg(module, msg, msg);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 555e7cb0b66..350f01b8093 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -215,7 +215,7 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
and LDB_SUCCESS on success
*/
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
- struct TDB_DATA tdb_key,
+ const struct TDB_DATA tdb_key,
struct ldb_message *msg,
unsigned int unpack_flags)
{
@@ -233,7 +233,6 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
ret = tdb_parse_record(ltdb->tdb, tdb_key,
ltdb_parse_data_unpack, &ctx);
- talloc_free(tdb_key.dptr);
if (ret == -1) {
ret = ltdb_err_map(tdb_error(ltdb->tdb));
@@ -266,13 +265,23 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
TDB_DATA tdb_key;
+
+ TALLOC_CTX *tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
- tdb_key = ltdb_key_dn(module, dn);
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OPERATIONS_ERROR;
}
ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
+
+ TALLOC_FREE(tdb_key_ctx);
+
if (ret != LDB_SUCCESS) {
return ret;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 6392201694e..f81fc054c3a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -164,9 +164,9 @@ bool ltdb_key_is_record(TDB_DATA key)
note that the key for a record can depend on whether the
dn refers to a case sensitive index record or not
*/
-TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn)
{
- struct ldb_context *ldb = ldb_module_get_ctx(module);
TDB_DATA key;
char *key_str = NULL;
const char *dn_folded = NULL;
@@ -188,7 +188,7 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
goto failed;
}
- key_str = talloc_strdup(ldb, "DN=");
+ key_str = talloc_strdup(mem_ctx, "DN=");
if (!key_str) {
goto failed;
}
@@ -238,15 +238,16 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
/*
form a TDB_DATA for a record key
- caller frees
+ caller frees mem_ctx, which may or may not have the key
+ as a child.
- note that the key for a record can depend on whether the
- dn refers to a case sensitive index record or not
+ note that the key for a record can depend on whether a
+ GUID index is in use, or the DN is used as the key
*/
-TDB_DATA ltdb_key_msg(struct ldb_module *module,
+TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg)
{
- return ltdb_key_dn(module, msg->dn);
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
}
/*
@@ -336,16 +337,22 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
TDB_DATA tdb_key, tdb_data;
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
+ TALLOC_CTX *tdb_key_ctx = talloc_new(module);
+
+ if (tdb_key_ctx == NULL) {
+ return ldb_module_oom(module);
+ }
- tdb_key = ltdb_key_msg(module, msg);
+ tdb_key = ltdb_key_msg(module, tdb_key_ctx, msg);
if (tdb_key.dptr == NULL) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
ret = ldb_pack_data(ldb_module_get_ctx(module),
msg, &ldb_data);
if (ret == -1) {
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
@@ -370,7 +377,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
}
done:
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
talloc_free(ldb_data.data);
return ret;
@@ -531,14 +538,20 @@ int ltdb_delete_noindex(struct ldb_module *module,
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA tdb_key;
int ret;
+ TALLOC_CTX *tdb_key_ctx = talloc_new(module);
+
+ if (tdb_key_ctx == NULL) {
+ return ldb_module_oom(module);
+ }
- tdb_key = ltdb_key_dn(module, msg->dn);
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, msg->dn);
if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
ret = tdb_delete(ltdb->tdb, tdb_key);
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
if (ret != 0) {
ret = ltdb_err_map(tdb_error(ltdb->tdb));
@@ -1187,13 +1200,13 @@ static int ltdb_rename(struct ltdb_context *ctx)
* Even in GUID index mode we use ltdb_key_dn() as we are
* trying to figure out if this is just a case rename
*/
- tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
+ tdb_key = ltdb_key_dn(module, msg, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key_old = ltdb_key_dn(module, req->op.rename.olddn);
+ tdb_key_old = ltdb_key_dn(module, msg, req->op.rename.olddn);
if (!tdb_key_old.dptr) {
talloc_free(msg);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 1e639e9a60a..f79a8d0dd66 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -144,8 +144,10 @@ int ltdb_unlock_read(struct ldb_module *module);
* index, the old DN index and a possible future ID=
*/
bool ltdb_key_is_record(TDB_DATA key);
-TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
-TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
+TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn);
+TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ const struct ldb_message *msg);
TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
struct ltdb_private *ltdb,
TALLOC_CTX *mem_ctx,
--
2.11.0
From eaa8581ca3ee162dc35469f2c592390951f1641d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:53:34 +1200
Subject: [PATCH 43/94] ldb_tdb: Use ltdb_key_msg() in ltdb_delete_noindex()
This allows the optional use of GUID based TDB key.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f81fc054c3a..82a31cfbe82 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -544,7 +544,7 @@ int ltdb_delete_noindex(struct ldb_module *module,
return ldb_module_oom(module);
}
- tdb_key = ltdb_key_dn(module, tdb_key_ctx, msg->dn);
+ tdb_key = ltdb_key_msg(module, tdb_key_ctx, msg);
if (!tdb_key.dptr) {
TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
--
2.11.0
From 8b53e34e07443d942e2a2fb8378846bc2a13968b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:11:13 +1200
Subject: [PATCH 44/94] ldb_tdb: Use the objectGUID (or similar) as the TDB key
in ltdb_key_msg()
When we have the full ldb_message we can read the objectGUID as the TDB key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 25 ++++++++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 82a31cfbe82..70be65f00bd 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -247,7 +247,30 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg)
{
- return ltdb_key_dn(module, mem_ctx, msg->dn);
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ TDB_DATA key;
+ const struct ldb_val *guid_val;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
+ }
+
+ if (ldb_dn_is_special(msg->dn)) {
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
+ }
+
+ guid_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (guid_val == NULL) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+
+ return ltdb_guid_to_key(module, ltdb, mem_ctx, guid_val);
+
}
/*
--
2.11.0
From 9c9d5ff960861c7bd5320b8b927b28adef16c26e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:12:30 +1200
Subject: [PATCH 45/94] ldb_tdb: Optionally use GUID index in ltdb_search_dn1()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 350f01b8093..249ecfeb810 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -271,11 +271,28 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return ldb_module_oom(module);
}
- /* form the key */
- tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
- if (!tdb_key.dptr) {
- TALLOC_FREE(tdb_key_ctx);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
+ if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ } else if (ldb_dn_is_special(dn)) {
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
+ if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ } else {
+ /* Look in the index to find the key for this DN */
+ ret = ltdb_key_dn_from_idx(module, ltdb, tdb_key_ctx,
+ dn, &tdb_key);
+ if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(tdb_key_ctx);
+ return ret;
+ }
}
ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
--
2.11.0
From 948029cab4312a5c7426fb13fa950841c9ce240c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 16:27:46 +1200
Subject: [PATCH 46/94] ldb_tdb: Do not query an index on the
GUID_index_attribute
The objectGUID (or similar) is already the record key, there is
no need to index it to itself.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f6dfb8fc3c9..3560aa64875 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -739,7 +739,32 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
dn_tree = *tree;
dn_tree.u.equality.attr = LTDB_IDXDN;
tree = &dn_tree;
+ } else if ((ltdb->cache->GUID_index_attribute != NULL) &&
+ (ldb_attr_cmp(tree->u.equality.attr,
+ ltdb->cache->GUID_index_attribute) == 0)) {
+ int ret;
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ list->dn = talloc_array(list, struct ldb_val, 1);
+ if (list->dn == NULL) {
+ ldb_module_oom(module);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ /*
+ * We need to go via the canonicalise_fn() to
+ * ensure we get the index in binary, rather
+ * than a string
+ */
+ ret = ltdb->GUID_index_syntax->canonicalise_fn(ldb,
+ list->dn,
+ &tree->u.equality.value,
+ &list->dn[0]);
+ if (ret != LDB_SUCCESS) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->count = 1;
+ return LDB_SUCCESS;
}
+
return ltdb_index_dn_simple(module, ltdb, tree, list);
}
--
2.11.0
From a027b6964aead7bec526c2e68a3a263f718b8359 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:44:34 +1200
Subject: [PATCH 47/94] ldb_tdb: Do not add an index for GUID_index_attribute
This would be pointless and we no longer query for it.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3560aa64875..90fd8994f83 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -624,6 +624,12 @@ static bool ltdb_is_indexed(struct ldb_module *module,
unsigned int i;
struct ldb_message_element *el;
+ if ((ltdb->cache->GUID_index_attribute != NULL) &&
+ (ldb_attr_cmp(attr,
+ ltdb->cache->GUID_index_attribute) == 0)) {
+ /* Implicity covered, this is the index key */
+ return false;
+ }
if (ldb->schema.index_handler_override) {
const struct ldb_schema_attribute *a
= ldb_schema_attribute_by_name(ldb, attr);
--
2.11.0
From 622bf4397751316840dea54ca362c6a652060482 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 07:15:50 +1200
Subject: [PATCH 48/94] ldb_tdb: Add ltdb_idx_to_key() and use it in
ltdb_index_filter()
This will allow a common point to parse index records into a TDB key,
allowing them to be a GUID or DN in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++------
lib/ldb/ldb_tdb/ldb_tdb.c | 28 ++++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
3 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 90fd8994f83..553b62e5da2 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1233,7 +1233,7 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
ldb = ldb_module_get_ctx(ac->module);
for (i = 0; i < dn_list->count; i++) {
- struct ldb_dn *dn;
+ TDB_DATA tdb_key;
int ret;
bool matched;
@@ -1242,16 +1242,17 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn = ldb_dn_from_ldb_val(msg, ldb, &dn_list->dn[i]);
- if (dn == NULL) {
- talloc_free(msg);
+ tdb_key = ltdb_idx_to_key(ac->module, ltdb,
+ ac, &dn_list->dn[i]);
+ if (tdb_key.dptr == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_search_dn1(ac->module, dn, msg,
+ ret = ltdb_search_key(ac->module, ltdb,
+ tdb_key, msg,
LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
- talloc_free(dn);
+ TALLOC_FREE(tdb_key.dptr);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* the record has disappeared? yes, this can happen */
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 70be65f00bd..bf3cddfbc1a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -236,6 +236,34 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
return key;
}
+TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val)
+{
+ TDB_DATA key;
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ struct ldb_dn *dn;
+
+ dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
+ if (dn == NULL) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ /* form the key */
+ key = ltdb_key_dn(module, mem_ctx, dn);
+ TALLOC_FREE(dn);
+ if (!key.dptr) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ return key;
+}
+
/*
form a TDB_DATA for a record key
caller frees mem_ctx, which may or may not have the key
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f79a8d0dd66..c40d39fd5e4 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -152,6 +152,10 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
struct ltdb_private *ltdb,
TALLOC_CTX *mem_ctx,
const struct ldb_val *guid_val);
+TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From 060317fd20e7ebabd068f23ec97fa4aa64b4de95 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 09:59:54 +1200
Subject: [PATCH 49/94] ldb_tdb: Trust the BASE and ONELEVEL index
This avoids re-checking the fetched DN against the scope
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 28 ++++++++++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 553b62e5da2..8579b01ed33 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1265,8 +1265,18 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ldb_match_msg_error(ldb, msg,
- ac->tree, ac->base, ac->scope, &matched);
+ /* We trust the index for SCOPE_ONELEVEL and SCOPE_BASE */
+ if ((ac->scope == LDB_SCOPE_ONELEVEL
+ && ltdb->cache->one_level_indexes)
+ || ac->scope == LDB_SCOPE_BASE) {
+ ret = ldb_match_message(ldb, msg, ac->tree,
+ ac->scope, &matched);
+ } else {
+ ret = ldb_match_msg_error(ldb, msg,
+ ac->tree, ac->base,
+ ac->scope, &matched);
+ }
+
if (ret != LDB_SUCCESS) {
talloc_free(msg);
return ret;
@@ -1354,6 +1364,11 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
switch (ac->scope) {
case LDB_SCOPE_BASE:
+ /*
+ * If we ever start to also load the index values for
+ * the tree, we must ensure we strictly intersect with
+ * this list, as we trust the BASE index
+ */
ret = ltdb_index_dn_base_dn(ac->module, ltdb,
ac->base, dn_list);
if (ret != LDB_SUCCESS) {
@@ -1367,6 +1382,11 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
+ /*
+ * If we ever start to also load the index values for
+ * the tree, we must ensure we strictly intersect with
+ * this list, as we trust the ONELEVEL index
+ */
ret = ltdb_index_dn_one(ac->module, ltdb, ac->base, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
@@ -1380,6 +1400,10 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
+ /*
+ * Here we load the index for the tree. We have no
+ * index for the subtree.
+ */
ret = ltdb_index_dn(ac->module, ltdb, ac->tree, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
--
2.11.0
From 48a72f62ea93137ad3bfbdb836388436664971ac Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:05:37 +1200
Subject: [PATCH 50/94] ldb_tdb: Optionally use GUID index values a direct TDB
keys
This connects the GUID based index records to GUID based TDB keys.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index bf3cddfbc1a..292c181cee0 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -245,6 +245,10 @@ TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct ldb_dn *dn;
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return ltdb_guid_to_key(module, ltdb, mem_ctx, idx_val);
+ }
+
dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
if (dn == NULL) {
errno = EINVAL;
--
2.11.0
From 040329015aff29b8f98671d1b7fc91708f6bafe5 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:57:57 +1200
Subject: [PATCH 51/94] ldb_tdb: Read from @INDEXLIST or an override if we are
using a GUID index
This allows all the previous patches to be enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/common/ldb_attributes.c | 14 ++++++++++++++
lib/ldb/include/ldb_module.h | 16 ++++++++++++++++
lib/ldb/include/ldb_private.h | 3 +++
lib/ldb/ldb_tdb/ldb_cache.c | 10 ++++++++++
4 files changed, 43 insertions(+)
diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c
index 98ec5a409ba..32f25fd0fe8 100644
--- a/lib/ldb/common/ldb_attributes.c
+++ b/lib/ldb/common/ldb_attributes.c
@@ -395,3 +395,17 @@ void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
ldb->schema.index_handler_override = true;
ldb->schema.one_level_indexes = one_level_indexes;
}
+
+/*
+ * set that the GUID index mode is in operation
+ *
+ * The caller must ensure the supplied strings do not go out of
+ * scope (they are typically constant memory).
+ */
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component)
+{
+ ldb->schema.GUID_index_attribute = GUID_index_attribute;
+ ldb->schema.GUID_index_dn_component = GUID_index_dn_component;
+}
diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index 71b407496f0..ffa6c2eedb8 100644
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -183,6 +183,22 @@ void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
bool one_level_indexes);
+/**
+
+ \param ldb The ldb context
+ \param GUID_index_attribute The globally attribute (eg objectGUID)
+ on each entry
+ \param GUID_index_attribute The DN component matching the
+ globally attribute on each entry (eg GUID)
+
+ The caller must ensure the supplied strings do not go out of
+ scope (they are typically constant memory).
+
+*/
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component);
+
/* A useful function to build comparison functions with */
int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx,
ldb_attr_handler_t canonicalise_fn,
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index ab215411a9a..f999f7530bf 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -98,6 +98,9 @@ struct ldb_schema {
*/
bool index_handler_override;
bool one_level_indexes;
+
+ const char *GUID_index_attribute;
+ const char *GUID_index_dn_component;
};
/*
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 137ddf97fe8..7bf5f156b45 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -243,6 +243,10 @@ static int ltdb_index_load(struct ldb_module *module,
*/
ltdb->cache->attribute_indexes = true;
ltdb->cache->one_level_indexes = ldb->schema.one_level_indexes;
+ ltdb->cache->GUID_index_attribute
+ = ldb->schema.GUID_index_attribute;
+ ltdb->cache->GUID_index_dn_component
+ = ldb->schema.GUID_index_dn_component;
return 0;
}
@@ -276,6 +280,12 @@ static int ltdb_index_load(struct ldb_module *module,
if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
ltdb->cache->attribute_indexes = true;
}
+ ltdb->cache->GUID_index_attribute
+ = ldb_msg_find_attr_as_string(ltdb->cache->indexlist,
+ LTDB_IDXGUID, NULL);
+ ltdb->cache->GUID_index_dn_component
+ = ldb_msg_find_attr_as_string(ltdb->cache->indexlist,
+ LTDB_IDX_DN_GUID, NULL);
return 0;
}
--
2.11.0
From 053635bdd013e0a968e9a5ab01a73e80ca6b2456 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 11:17:59 +1200
Subject: [PATCH 52/94] ldb_tdb: Add improved error strings on
ltdb_key_dn_from_idx() failure
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8579b01ed33..de91377e95f 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -316,6 +316,14 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
return LDB_ERR_NO_SUCH_OBJECT;
}
if (list->count > 1) {
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ __location__
+ ": Failed to read DN index "
+ "against %s for %s: too many "
+ "values (%u > 1)",
+ ltdb->cache->GUID_index_attribute,
+ dn_str, list->count);
return LDB_ERR_CONSTRAINT_VIOLATION;
}
--
2.11.0
From 0496b1a4276e4d78b5e24fe181f1bcd6765774c1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 12:59:50 +1200
Subject: [PATCH 53/94] ldb_tdb: Improve debugging in ltdb_modify_index_dn() on
casefold failure
This is unlikely, but when it happens it will be really painful to debug.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index de91377e95f..b77522c703c 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1632,6 +1632,15 @@ static int ltdb_modify_index_dn(struct ldb_module *module,
val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
if (val.data == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ __location__
+ ": Failed to modify %s "
+ "against %s in %s: failed "
+ "to get casefold DN",
+ index,
+ ltdb->cache->GUID_index_attribute,
+ dn_str);
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From 7b9cfa8c41d1d4ca09d634c8d47e43d6db83a97f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 14:24:44 +1200
Subject: [PATCH 54/94] ldb_tdb: Add unique index checking for @IDXDN
This will give us errors earlier if the index code becomes broken
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b77522c703c..4e7b1012771 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1478,8 +1478,13 @@ static int ltdb_index_add1(struct ldb_module *module,
return ret;
}
+ /*
+ * Check for duplicates in unique indexes and for the @IDXDN
+ * DN -> GUID record
+ */
if (list->count > 0 &&
- a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX) {
+ (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
+ ldb_attr_cmp(el->name, LTDB_IDXDN) == 0)) {
/*
* We do not want to print info about a possibly
* confidential DN that the conflict was with in the
--
2.11.0
From 9212c925da5b5fdc7bfac87716b520c5da6568aa Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 12:37:51 +1200
Subject: [PATCH 55/94] ldb_tdb: Re-add of both existing DN and GUID must gives
LDB_ERR_ENTRY_ALREADY_EXISTS
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 292c181cee0..95b74bef391 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -525,6 +525,24 @@ static int ltdb_add_internal(struct ldb_module *module,
ret = ltdb_store(module, msg, TDB_INSERT);
if (ret != LDB_SUCCESS) {
+ /*
+ * Try really hard to get the right error code for
+ * a re-add situation, as this can matter!
+ */
+ if (ret == LDB_ERR_CONSTRAINT_VIOLATION) {
+ int ret2;
+ struct ldb_dn *dn2 = NULL;
+ TALLOC_CTX *mem_ctx = talloc_new(module);
+ if (mem_ctx == NULL) {
+ return ldb_module_operr(module);
+ }
+ ret2 = ltdb_search_base(module, module,
+ msg->dn, &dn2);
+ TALLOC_FREE(mem_ctx);
+ if (ret2 == LDB_SUCCESS) {
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ }
+ }
if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
ldb_asprintf_errstring(ldb,
"Entry %s already exists",
--
2.11.0
From cda98419fe0e3455ae25d425d6f80ff0d32358ce Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 14:25:46 +1200
Subject: [PATCH 56/94] ldb_tdb: Change error code on unique index violation
The LDB_ERR_ENTRY_ALREADY_EXISTS error code is detected in repl_meta_data as indicating
that the DN exists, and that a conflict record should be created.
This is really a constraint violation, not a duplicate record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 4e7b1012771..054bb8f18e9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1502,7 +1502,7 @@ static int ltdb_index_add1(struct ldb_module *module,
el->name,
ldb_dn_get_linearized(msg->dn));
talloc_free(list);
- return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ return LDB_ERR_CONSTRAINT_VIOLATION;
}
/* overallocate the list a bit, to reduce the number of
--
2.11.0
From 339ba35360db0ef3a821681e0708231fb8e83534 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 15:35:32 +1200
Subject: [PATCH 57/94] ldb_tdb: sort GUID index list at add time with an
insertion sort
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 154 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 152 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 054bb8f18e9..055cf38fb6e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1,4 +1,122 @@
/*
+ Unix SMB/CIFS implementation.
+
+ a generic binary search macro
+
+ Copyright (C) Andrew Tridgell 2009
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _BINSEARCH_H
+#define _BINSEARCH_H
+
+/* a binary array search, where the array is an array of pointers to structures,
+ and we want to find a match for 'target' on 'field' in those structures.
+
+ Inputs:
+ array: base pointer to an array of structures
+ arrray_size: number of elements in the array
+ field: the name of the field in the structure we are keying off
+ target: the field value we are looking for
+ comparison_fn: the comparison function
+ result: where the result of the search is put
+
+ if the element is found, then 'result' is set to point to the found array element. If not,
+ then 'result' is set to NULL.
+
+ The array is assumed to be sorted by the same comparison_fn as the
+ search (with, for example, qsort)
+ */
+#define BINARY_ARRAY_SEARCH_P(array, array_size, field, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i]->field); \
+ if (_r == 0) { (result) = array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+/*
+ like BINARY_ARRAY_SEARCH_P, but assumes that the array is an array
+ of structures, rather than pointers to structures
+
+ result points to the found structure, or NULL
+ */
+#define BINARY_ARRAY_SEARCH(array, array_size, field, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i].field); \
+ if (_r == 0) { (result) = &array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+/*
+ like BINARY_ARRAY_SEARCH_P, but assumes that the array is an array
+ of elements, rather than pointers to structures
+
+ result points to the found structure, or NULL
+ */
+#define BINARY_ARRAY_SEARCH_V(array, array_size, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i]); \
+ if (_r == 0) { (result) = &array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+
+/*
+ like BINARY_ARRAY_SEARCH_V, but if an exact result is not found, the 'next'
+ argument will point to the element after the place where the exact result
+ would have been. If an exact result is found, 'next' will be NULL. If the
+ target is beyond the end of the list, both 'exact' and 'next' will be NULL.
+ Unlike other binsearch macros, where there are several elements that compare
+ the same, the exact result will always point to the first one.
+
+ If you don't care to distinguish between the 'greater than' and 'equals'
+ cases, you can use the same pointer for both 'exact' and 'next'.
+
+ As with all the binsearch macros, the comparison function is always called
+ with the search term first.
+ */
+#define BINARY_ARRAY_SEARCH_GTE(array, array_size, target, comparison_fn, \
+ exact, next) do { \
+ int32_t _b, _e; \
+ (exact) = NULL; (next) = NULL; \
+ if ((array_size) > 0) { \
+ for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b + _e) / 2; \
+ int _r = comparison_fn(target, &array[_i]); \
+ if (_r == 0) { \
+ (exact) = &array[_i]; \
+ _e = _i - 1; \
+ } else if (_r < 0) { _e = _i - 1; \
+ } else { _b = _i + 1; } \
+ } \
+ if ((exact) == NULL &&_b < (array_size)) { \
+ (next) = &array[_b]; \
+ } } } while (0)
+
+#endif
+
+/*
ldb database library
Copyright (C) Andrew Tridgell 2004-2009
@@ -87,6 +205,22 @@ static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1,
return memcmp(v1->data, v2->data, v1->length);
}
+/*
+ see if two ldb_val structures contain exactly the same data
+ return -1 or 1 for a mismatch, 0 for match
+*/
+static int ldb_val_equal_exact_ordered(const struct ldb_val v1,
+ const struct ldb_val *v2)
+{
+ if (v1.length > v2->length) {
+ return -1;
+ }
+ if (v1.length < v2->length) {
+ return 1;
+ }
+ return memcmp(v1.data, v2->data, v1.length);
+}
+
/*
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
@@ -1525,6 +1659,7 @@ static int ltdb_index_add1(struct ldb_module *module,
list->dn[list->count].length = strlen(dn_str);
} else {
const struct ldb_val *key_val;
+ struct ldb_val *exact = NULL, *next = NULL;
key_val = ldb_msg_find_ldb_val(msg,
ltdb->cache->GUID_index_attribute);
if (key_val == NULL) {
@@ -1536,8 +1671,23 @@ static int ltdb_index_add1(struct ldb_module *module,
talloc_free(list);
return ldb_module_operr(module);
}
- list->dn[list->count] = ldb_val_dup(list->dn, key_val);
- if (list->dn[list->count].data == NULL) {
+
+ BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
+ *key_val, ldb_val_equal_exact_ordered,
+ exact, next);
+ if (exact != NULL) {
+ talloc_free(list);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ if (next == NULL) {
+ next = &list->dn[list->count];
+ } else {
+ memmove(&next[1], next,
+ sizeof(*next) * (list->count - (next - list->dn)));
+ }
+ *next = ldb_val_dup(list->dn, key_val);
+ if (next->data == NULL) {
talloc_free(list);
return ldb_module_operr(module);
}
--
2.11.0
From 389edf2f24f733dda4f8c497870439996ac2763e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:51:30 +1200
Subject: [PATCH 58/94] ldb_tdb: Sort inputs to list_union()
This allows us to merge the lists finding common values.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 38 +++++++++++++++++++++++++++++++++++---
1 file changed, 35 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 055cf38fb6e..a14181513d3 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -170,6 +170,9 @@ static int ltdb_index_dn_base_dn(struct ldb_module *module,
struct ldb_dn *base_dn,
struct dn_list *dn_list);
+static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
+ struct dn_list *list);
+
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
*/
@@ -852,7 +855,9 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
}
-static bool list_union(struct ldb_context *, struct dn_list *, const struct dn_list *);
+static bool list_union(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
+ struct dn_list *list, struct dn_list *list2);
/*
return a list of dn's that might match a leaf indexed search
@@ -990,7 +995,8 @@ static bool list_intersect(struct ldb_context *ldb,
list = list | list2
*/
static bool list_union(struct ldb_context *ldb,
- struct dn_list *list, const struct dn_list *list2)
+ struct ltdb_private *ltdb,
+ struct dn_list *list, struct dn_list *list2)
{
struct ldb_val *dn3;
@@ -1011,6 +1017,13 @@ static bool list_union(struct ldb_context *ldb,
return true;
}
+ /*
+ * Sort the lists (if not in GUID DN mode) so we can do
+ * the de-duplication during the merge
+ */
+ ltdb_dn_list_sort(ltdb, list);
+ ltdb_dn_list_sort(ltdb, list2);
+
dn3 = talloc_array(list, struct ldb_val, list->count + list2->count);
if (!dn3) {
ldb_oom(ldb);
@@ -1073,7 +1086,7 @@ static int ltdb_index_dn_or(struct ldb_module *module,
return ret;
}
- if (!list_union(ldb, list, list2)) {
+ if (!list_union(ldb, ltdb, list, list2)) {
talloc_free(list2);
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1453,6 +1466,25 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
}
/*
+ sort a DN list
+ */
+static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
+ struct dn_list *list)
+{
+ if (list->count < 2) {
+ return;
+ }
+
+ /* We know the list is sorted when using the GUID index */
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return;
+ }
+
+ TYPESAFE_QSORT(list->dn, list->count,
+ ldb_val_equal_exact_for_qsort);
+}
+
+/*
remove any duplicated entries in a indexed result
*/
static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
--
2.11.0
From 86b94335269c179332a20626536f3b2ce679e1c4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:07:45 +1200
Subject: [PATCH 59/94] ldb_tdb: Rework list_union to not return duplicates,
and keep sort order
This allows the binary search to still operate on the list, even after
a or operator in the search expression
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 35 +++++++++++++++++++++++++++++++----
1 file changed, 31 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index a14181513d3..387898304cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -999,6 +999,7 @@ static bool list_union(struct ldb_context *ldb,
struct dn_list *list, struct dn_list *list2)
{
struct ldb_val *dn3;
+ unsigned int i = 0, j = 0, k = 0;
if (list2->count == 0) {
/* X | 0 == X */
@@ -1030,12 +1031,38 @@ static bool list_union(struct ldb_context *ldb,
return false;
}
- /* we allow for duplicates here, and get rid of them later */
- memcpy(dn3, list->dn, sizeof(list->dn[0])*list->count);
- memcpy(dn3+list->count, list2->dn, sizeof(list2->dn[0])*list2->count);
+ while (i < list->count || j < list2->count) {
+ int cmp;
+ if (i >= list->count) {
+ cmp = 1;
+ } else if (j >= list2->count) {
+ cmp = -1;
+ } else {
+ cmp = ldb_val_equal_exact_ordered(list->dn[i],
+ &list2->dn[j]);
+ }
+
+ if (cmp < 0) {
+ /* Take list */
+ dn3[k] = list->dn[i];
+ i++;
+ k++;
+ } else if (cmp > 0) {
+ /* Take list2 */
+ dn3[k] = list2->dn[j];
+ j++;
+ k++;
+ } else {
+ /* Equal, take list */
+ dn3[k] = list->dn[i];
+ i++;
+ j++;
+ k++;
+ }
+ }
list->dn = dn3;
- list->count += list2->count;
+ list->count = k;
return true;
}
--
2.11.0
From f5a382a745c64a765e4cdfb648e99c3563397a77 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 15:51:19 +1200
Subject: [PATCH 60/94] ldb_tdb: Use a binary search to speed up
ltdb_dn_list_find_val()
This only works if we have the GUID index format, as otherwise these are unsorted.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 26 ++++++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 387898304cc..9d85494f4d0 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -236,12 +236,30 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
const struct ldb_val *v)
{
unsigned int i;
- for (i=0; i<list->count; i++) {
- if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
- return i;
+ struct ldb_val *exact = NULL, *next = NULL;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ for (i=0; i<list->count; i++) {
+ if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
+ return i;
+ }
}
+ return -1;
+ }
+
+ BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
+ *v, ldb_val_equal_exact_ordered,
+ exact, next);
+ if (exact == NULL) {
+ return -1;
}
- return -1;
+ /* Not required, but keeps the compiler quiet */
+ if (next != NULL) {
+ return -1;
+ }
+
+ i = exact - list->dn;
+ return i;
}
/*
--
2.11.0
From 9d82b0915c7f82903529fc2794812c1312786841 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:16:56 +1200
Subject: [PATCH 61/94] ldb_tdb: Use the binary search more efficiently in
list_intersect()
This change ensures we walk the short list and look up into the longer of the two lists.
ltdb_dn_list_find_val() will do a binary search for the GUID case.
Before GUID indexes this was O(n*m), now it is O(n*log(m)).
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 9d85494f4d0..0a9e1167dd5 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -948,6 +948,7 @@ static bool list_intersect(struct ldb_context *ldb,
struct ltdb_private *ltdb,
struct dn_list *list, const struct dn_list *list2)
{
+ const struct dn_list *short_list, *long_list;
struct dn_list *list3;
unsigned int i;
@@ -980,6 +981,14 @@ static bool list_intersect(struct ldb_context *ldb,
return true;
}
+ if (list->count > list2->count) {
+ short_list = list2;
+ long_list = list;
+ } else {
+ short_list = list;
+ long_list = list2;
+ }
+
list3 = talloc_zero(list, struct dn_list);
if (list3 == NULL) {
return false;
@@ -992,10 +1001,11 @@ static bool list_intersect(struct ldb_context *ldb,
}
list3->count = 0;
- for (i=0;i<list->count;i++) {
- if (ltdb_dn_list_find_val(ltdb, list2,
- &list->dn[i]) != -1) {
- list3->dn[list3->count] = list->dn[i];
+ for (i=0;i<short_list->count;i++) {
+ /* For the GUID index case, this is a binary search */
+ if (ltdb_dn_list_find_val(ltdb, long_list,
+ &short_list->dn[i]) != -1) {
+ list3->dn[list3->count] = short_list->dn[i];
list3->count++;
}
}
--
2.11.0
From 21f55ac5c6635a3dd030dc13684bb746c61f0ae7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:17:56 +1200
Subject: [PATCH 62/94] ldb_tdb: Reduce memory consumption in list_intersect()
We will never have more results than is in either list or list2.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 0a9e1167dd5..0418988b88d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -994,7 +994,8 @@ static bool list_intersect(struct ldb_context *ldb,
return false;
}
- list3->dn = talloc_array(list3, struct ldb_val, list->count);
+ list3->dn = talloc_array(list3, struct ldb_val,
+ MIN(list->count, list2->count));
if (!list3->dn) {
talloc_free(list3);
return false;
--
2.11.0
From 052ed0098f76a76003fe58e619bd3681a9066236 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:55:01 +1200
Subject: [PATCH 63/94] ldb_tdb: Duplicate values are no longer permitted in
the index
By removing the qsort() we avoid work.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 29 -----------------------------
1 file changed, 29 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 0418988b88d..6043b16a6ad 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1541,34 +1541,6 @@ static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
}
/*
- remove any duplicated entries in a indexed result
- */
-static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
-{
- unsigned int i, new_count;
-
- if (list->count < 2) {
- return;
- }
-
- TYPESAFE_QSORT(list->dn, list->count,
- ldb_val_equal_exact_for_qsort);
-
- new_count = 1;
- for (i=1; i<list->count; i++) {
- if (ldb_val_equal_exact(&list->dn[i],
- &list->dn[new_count-1]) == 0) {
- if (new_count != i) {
- list->dn[new_count] = list->dn[i];
- }
- new_count++;
- }
- }
-
- list->count = new_count;
-}
-
-/*
search the database with a LDAP-like expression using indexes
returns -1 if an indexed search is not possible, in which
case the caller should call ltdb_search_full()
@@ -1639,7 +1611,6 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return ret;
}
- ltdb_dn_list_remove_duplicates(dn_list);
break;
}
--
2.11.0
From 4afc340597d8cba0ec5f2778ecec4d8eb1701d6c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 12:06:48 +1200
Subject: [PATCH 64/94] ldb: Add an unused objectGUID to each record in
SearchTests
This will then be used by the GUID index tests.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 81 +++++++++++++++++++++++++++++++--------------
1 file changed, 56 insertions(+), 25 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index b95266b0384..1db2c5ca003 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -616,79 +616,110 @@ class SearchTests(TestCase):
self.filename = os.path.join(self.testdir, "search_test.ldb")
self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
- self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
+ # Note that we can't use the name objectGUID here, as we
+ # want to stay clear of the objectGUID handler in LDB and
+ # instead use just the 16 bytes raw, which we just keep
+ # to printable chars here for ease of handling.
+
+ self.l.add({"dn": "DC=SAMBA,DC=ORG",
+ "name": b"samba.org",
+ "objectUUID": b"0123456789abcddf"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
"name": b"Admins",
- "x": "z", "y": "a"})
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG",
"name": b"Users",
- "x": "z", "y": "a"})
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG",
"name": b"OU #1",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde3"})
self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG",
"name": b"OU #2",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde4"})
self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG",
"name": b"OU #3",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde5"})
self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG",
"name": b"OU #4",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde6"})
self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG",
"name": b"OU #5",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde7"})
self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG",
"name": b"OU #6",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde8"})
self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG",
"name": b"OU #7",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde9"})
self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG",
"name": b"OU #8",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde0"})
self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG",
"name": b"OU #9",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdea"})
self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdeb"})
self.l.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdec"})
self.l.add({"dn": "OU=OU12,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "b"})
+ "x": "y", "y": "b",
+ "objectUUID": b"0123456789abcded"})
self.l.add({"dn": "OU=OU13,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcdee"})
self.l.add({"dn": "OU=OU14,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd01"})
self.l.add({"dn": "OU=OU15,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd02"})
self.l.add({"dn": "OU=OU16,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd03"})
self.l.add({"dn": "OU=OU17,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd04"})
self.l.add({"dn": "OU=OU18,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd05"})
self.l.add({"dn": "OU=OU19,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd06"})
self.l.add({"dn": "OU=OU20,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd07"})
self.l.add({"dn": "OU=OU21,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "c"})
+ "x": "x", "y": "c",
+ "objectUUID": b"0123456789abcd08"})
self.l.add({"dn": "OU=OU22,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "c"})
+ "x": "x", "y": "c",
+ "objectUUID": b"0123456789abcd09"})
def test_base(self):
"""Testing a search"""
--
2.11.0
From 67f4d3d09e90ade1b499411811da4e5fa9b739c6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 15:34:31 +1200
Subject: [PATCH 65/94] ldb: Also test the new GUID index mode
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 1db2c5ca003..cc707edab3e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -934,6 +934,17 @@ class IndexedSearchTests(SearchTests):
"@IDXATTR": [b"x", b"y", b"ou"],
"@IDXONE": [b"1"]})
+class GUIDIndexedSearchTests(SearchTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedSearchTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]})
+
class DnTests(TestCase):
--
2.11.0
From 8a168cd05ee27060d5e0183cd15d9953db1414d4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:22:27 +1200
Subject: [PATCH 66/94] ldb_tdb: Add tests for add/modify with the GUID index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 84 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index cc707edab3e..bbdb775ceb3 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -945,6 +945,90 @@ class GUIDIndexedSearchTests(SearchTests):
"@IDXGUID": [b"objectUUID"],
"@IDX_DN_GUID": [b"GUID"]})
+class AddModifyTests(TestCase):
+ def tearDown(self):
+ shutil.rmtree(self.testdir)
+ super(AddModifyTests, self).tearDown()
+
+ def setUp(self):
+ super(AddModifyTests, self).setUp()
+ self.testdir = tempdir()
+ self.filename = os.path.join(self.testdir, "add_test.ldb")
+ self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l.add({"dn": "DC=SAMBA,DC=ORG",
+ "name": b"samba.org",
+ "objectUUID": b"0123456789abcdef"})
+
+ def test_add_dup(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ try:
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ self.fail("Should have failed adding dupliate entry")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ def test_add_del_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.delete("OU=DUP,DC=SAMBA,DC=ORG")
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+ def test_add_move_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+
+class IndexedAddModifyTests(AddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(IndexedAddModifyTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"]})
+
+class GUIDIndexedAddModifyTests(AddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedAddModifyTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]})
+
+ def test_duplicate_GUID(self):
+ try:
+ self.l.add({"dn": "OU=DUPGUID,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcdef"})
+ self.fail("Should have failed adding dupliate GUID")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+
class DnTests(TestCase):
--
2.11.0
From bda8381b8780531aa86866029fc1213a4b93fd88 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 28 Aug 2017 15:37:28 +1200
Subject: [PATCH 67/94] ldb_tdb: Use the DN extracted from the DB to filter the
message later
This should ensure that the upper or lower case the user chooses does not impact
on the filtering, at least for database that have checkBaseOnSearch set.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 38 +++++++++++++++++++++++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.c | 6 ++++--
lib/ldb/ldb_tdb/ldb_tdb.h | 5 ++++-
3 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 249ecfeb810..761d33ed6d6 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -113,7 +113,10 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_search_base(struct ldb_module *module,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ struct ldb_dn **ret_dn)
{
int exists;
int ret;
@@ -137,16 +140,34 @@ int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
ret = ltdb_search_dn1(module, dn,
msg,
- LDB_UNPACK_DATA_FLAG_NO_DN|
LDB_UNPACK_DATA_FLAG_NO_ATTRS);
- talloc_free(msg);
if (ret == LDB_SUCCESS) {
+ const char *dn_linearized
+ = ldb_dn_get_linearized(dn);
+ const char *msg_dn_linearlized
+ = ldb_dn_get_linearized(msg->dn);
+
+ if (strcmp(dn_linearized, msg_dn_linearlized) == 0) {
+ /*
+ * Re-use the full incoming DN for
+ * subtree checks
+ */
+ *ret_dn = dn;
+ } else {
+ /*
+ * Use the string DN from the unpack, so that
+ * we have a case-exact match of the base
+ */
+ *ret_dn = talloc_steal(mem_ctx, msg->dn);
+ }
exists = true;
} else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
exists = false;
} else {
+ talloc_free(msg);
return ret;
}
+ talloc_free(msg);
if (exists) {
return LDB_SUCCESS;
}
@@ -721,8 +742,15 @@ int ltdb_search(struct ltdb_context *ctx)
return ret;
} else if (ltdb->check_base) {
- /* This database has been marked as 'checkBaseOnSearch', so do a spot check of the base dn */
- ret = ltdb_search_base(module, req->op.search.base);
+ /*
+ * This database has been marked as
+ * 'checkBaseOnSearch', so do a spot check of the base
+ * dn. Also optimise the subsequent filter by filling
+ * in the ctx->base to be exactly case correct
+ */
+ ret = ltdb_search_base(module, ctx,
+ req->op.search.base,
+ &ctx->base);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
ldb_asprintf_errstring(ldb,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 95b74bef391..589050642e6 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1246,6 +1246,7 @@ static int ltdb_rename(struct ltdb_context *ctx)
struct ldb_message *msg;
int ret = LDB_SUCCESS;
TDB_DATA tdb_key, tdb_key_old;
+ struct ldb_dn *db_dn;
ldb_request_set_state(req, LDB_ASYNC_PENDING);
@@ -1292,8 +1293,9 @@ static int ltdb_rename(struct ltdb_context *ctx)
*/
if (tdb_key_old.dsize != tdb_key.dsize
|| memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
- ret = ltdb_search_base(module,
- req->op.rename.newdn);
+ ret = ltdb_search_base(module, msg,
+ req->op.rename.newdn,
+ &db_dn);
if (ret == LDB_SUCCESS) {
ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
} else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index c40d39fd5e4..0e6e9797d4b 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -126,7 +126,10 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
-int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn);
+int ltdb_search_base(struct ldb_module *module,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ struct ldb_dn **ret_dn);
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
struct TDB_DATA tdb_key,
struct ldb_message *msg,
--
2.11.0
From 0d20608802352640d22a35e66f257e5082f69eda Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 10:40:22 +1200
Subject: [PATCH 68/94] ldb: Add tests for base DN of a different case
This ensures we cover the case where the DN does not match the DB exactly
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index bbdb775ceb3..4c35e44ee28 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -616,6 +616,9 @@ class SearchTests(TestCase):
self.filename = os.path.join(self.testdir, "search_test.ldb")
self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l.add({"dn": "@ATTRIBUTES",
+ "DC": "CASE_INSENSITIVE"})
+
# Note that we can't use the name objectGUID here, as we
# want to stay clear of the objectGUID handler in LDB and
# instead use just the 16 bytes raw, which we just keep
@@ -728,6 +731,13 @@ class SearchTests(TestCase):
scope=ldb.SCOPE_BASE)
self.assertEqual(len(res11), 1)
+ def test_base_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="OU=OU11,DC=samba,DC=org",
+ scope=ldb.SCOPE_BASE)
+ self.assertEqual(len(res11), 1)
+
def test_base_or(self):
"""Testing a search"""
@@ -804,6 +814,14 @@ class SearchTests(TestCase):
expression="(&(x=y)(|(y=b)(y=c)))")
self.assertEqual(len(res11), 1)
+ def test_subtree_and2_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="DC=samba,DC=org",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(&(x=y)(|(y=b)(y=c)))")
+ self.assertEqual(len(res11), 1)
+
def test_subtree_or(self):
"""Testing a search"""
@@ -860,6 +878,14 @@ class SearchTests(TestCase):
expression="(|(x=y)(y=b))")
self.assertEqual(len(res11), 20)
+ def test_one_or2_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="DC=samba,DC=org",
+ scope=ldb.SCOPE_ONELEVEL,
+ expression="(|(x=y)(y=b))")
+ self.assertEqual(len(res11), 20)
+
def test_subtree_and_or(self):
"""Testing a search"""
--
2.11.0
From 32e6f3707d29d3dc48a34f778df68bd8bfb80f47 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 30 Aug 2017 13:18:20 +1200
Subject: [PATCH 69/94] ldb_tdb: Optimise ltdb_search_and_return_base() to
re-use casefolding
The casefolding of a DN is one of the more expensive and pointless things in LDB
operation. The ldb_dn abstraction works hard to avoid duplicating this work, but
we can work harder to save that information.
Here we copy in the DN, that has been casefolded already for the index,
and keep that as the returned DN, after stripping any extended components.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_search.c | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 761d33ed6d6..07757f86f3b 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -593,6 +593,8 @@ static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
{
struct ldb_message *msg, *filtered_msg;
struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
+ const char *dn_linearized;
+ const char *msg_dn_linearlized;
int ret;
bool matched;
@@ -642,9 +644,32 @@ static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
return LDB_SUCCESS;
}
- /* filter the attributes that the user wants */
+ dn_linearized = ldb_dn_get_linearized(ctx->base);
+ msg_dn_linearlized = ldb_dn_get_linearized(msg->dn);
+
+ if (strcmp(dn_linearized, msg_dn_linearlized) == 0) {
+ /*
+ * If the DN is exactly the same string, then
+ * re-use the full incoming DN for the
+ * returned result, as it has already been
+ * casefolded
+ */
+ msg->dn = ctx->base;
+ }
+
+ /*
+ * filter the attributes that the user wants.
+ *
+ * This copies msg->dn including the casefolding, so the above
+ * assignment is safe
+ */
ret = ltdb_filter_attrs(ctx, msg, ctx->attrs, &filtered_msg);
+ /*
+ * Remove any extended components possibly copied in from
+ * msg->dn, we just want the casefold components
+ */
+ ldb_dn_remove_extended_components(filtered_msg->dn);
talloc_free(msg);
if (ret == -1) {
--
2.11.0
From 135cf80144c47c558869eaf0bb191c3bbd929b88 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:04:43 +1200
Subject: [PATCH 70/94] ldb_tdb: Move constants into ldb_tdb.h
This helps ensure we keep these all in sync.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 6043b16a6ad..8f558195e10 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -408,7 +408,6 @@ normal_index:
list->count = el->num_values;
} else {
unsigned int i;
- static const size_t GUID_val_size = 16;
if (version != LTDB_GUID_INDEXING_VERSION) {
/* This is quite likely during the DB startup
on first upgrade to using a GUID index */
@@ -425,11 +424,11 @@ normal_index:
return LDB_ERR_OPERATIONS_ERROR;
}
- if ((el->values[0].length % GUID_val_size) != 0) {
+ if ((el->values[0].length % LTDB_GUID_SIZE) != 0) {
return LDB_ERR_OPERATIONS_ERROR;
}
- list->count = el->values[0].length / GUID_val_size;
+ list->count = el->values[0].length / LTDB_GUID_SIZE;
list->dn = talloc_array(list, struct ldb_val, list->count);
/*
@@ -438,8 +437,9 @@ normal_index:
*/
talloc_steal(list->dn, msg);
for (i = 0; i < list->count; i++) {
- list->dn[i].data = &el->values[0].data[i * GUID_val_size];
- list->dn[i].length = GUID_val_size;
+ list->dn[i].data
+ = &el->values[0].data[i * LTDB_GUID_SIZE];
+ list->dn[i].length = LTDB_GUID_SIZE;
}
}
--
2.11.0
From c5597fca70917773c2392fce7e253a05e2089971 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:06:15 +1200
Subject: [PATCH 71/94] ldb_tdb: Avoid allocation of a DN between the GUID
index and the DB lookup
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 28 +++++++++-----
lib/ldb/ldb_tdb/ldb_search.c | 34 ++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.c | 90 +++++++++++++++++++++++++-------------------
lib/ldb/ldb_tdb/ldb_tdb.h | 18 +++++----
4 files changed, 106 insertions(+), 64 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8f558195e10..78675e8ae56 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -479,12 +479,15 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
"values (%u > 1)",
ltdb->cache->GUID_index_attribute,
dn_str, list->count);
+ TALLOC_FREE(list);
return LDB_ERR_CONSTRAINT_VIOLATION;
}
- *tdb_key = ltdb_guid_to_key(module, ltdb,
- mem_ctx, &list->dn[0]);
- if (tdb_key->dptr == NULL) {
+ /* The tdb_key memory is allocated by the caller */
+ ret = ltdb_guid_to_key(module, ltdb,
+ &list->dn[0], tdb_key);
+
+ if (ret != LDB_SUCCESS) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1444,7 +1447,11 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
ldb = ldb_module_get_ctx(ac->module);
for (i = 0; i < dn_list->count; i++) {
- TDB_DATA tdb_key;
+ uint8_t guid_key[LTDB_GUID_KEY_SIZE];
+ TDB_DATA tdb_key = {
+ .dptr = guid_key,
+ .dsize = sizeof(guid_key)
+ };
int ret;
bool matched;
@@ -1453,17 +1460,20 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key = ltdb_idx_to_key(ac->module, ltdb,
- ac, &dn_list->dn[i]);
- if (tdb_key.dptr == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
+ ret = ltdb_idx_to_key(ac->module, ltdb,
+ ac, &dn_list->dn[i],
+ &tdb_key);
+ if (ret != LDB_SUCCESS) {
+ return ret;
}
ret = ltdb_search_key(ac->module, ltdb,
tdb_key, msg,
LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
- TALLOC_FREE(tdb_key.dptr);
+ if (tdb_key.dptr != guid_key) {
+ TALLOC_FREE(tdb_key.dptr);
+ }
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* the record has disappeared? yes, this can happen */
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 07757f86f3b..0af230f219b 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -285,14 +285,19 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
- TDB_DATA tdb_key;
-
- TALLOC_CTX *tdb_key_ctx = talloc_new(msg);
- if (!tdb_key_ctx) {
- return ldb_module_oom(module);
- }
+ uint8_t guid_key[LTDB_GUID_KEY_SIZE];
+ TDB_DATA tdb_key = {
+ .dptr = guid_key,
+ .dsize = sizeof(guid_key)
+ };
+ TALLOC_CTX *tdb_key_ctx = NULL;
if (ltdb->cache->GUID_index_attribute == NULL) {
+ tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
@@ -300,6 +305,11 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return LDB_ERR_OPERATIONS_ERROR;
}
} else if (ldb_dn_is_special(dn)) {
+ tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
@@ -307,11 +317,17 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return LDB_ERR_OPERATIONS_ERROR;
}
} else {
- /* Look in the index to find the key for this DN */
- ret = ltdb_key_dn_from_idx(module, ltdb, tdb_key_ctx,
+ /*
+ * Look in the index to find the key for this DN.
+ *
+ * the tdb_key memory is allocated above, msg is just
+ * used for internal memory.
+ *
+ */
+ ret = ltdb_key_dn_from_idx(module, ltdb,
+ msg,
dn, &tdb_key);
if (ret != LDB_SUCCESS) {
- TALLOC_FREE(tdb_key_ctx);
return ret;
}
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 589050642e6..b4ce2470352 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -210,62 +210,58 @@ failed:
return key;
}
-TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *GUID_val)
+/* The caller is to provide a correctly sized key */
+int ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_val *GUID_val,
+ TDB_DATA *key)
{
- TDB_DATA key;
- const char *GUID_prefix = "GUID=";
- const int GUID_prefix_len = strlen(GUID_prefix);
-
- key.dptr = talloc_size(mem_ctx,
- GUID_val->length+GUID_prefix_len);
+ const char *GUID_prefix = LTDB_GUID_KEY_PREFIX;
+ const int GUID_prefix_len = sizeof(LTDB_GUID_KEY_PREFIX) - 1;
- if (key.dptr == NULL) {
- errno = ENOMEM;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ if (key->dsize != (GUID_val->length+GUID_prefix_len)) {
+ return LDB_ERR_OPERATIONS_ERROR;
}
- memcpy(key.dptr, "GUID=", GUID_prefix_len);
- memcpy(&key.dptr[GUID_prefix_len],
- GUID_val->data, GUID_val->length);
- key.dsize = talloc_get_size(key.dptr);
- return key;
+ memcpy(key->dptr, GUID_prefix, GUID_prefix_len);
+ memcpy(&key->dptr[GUID_prefix_len],
+ GUID_val->data, GUID_val->length);
+ return LDB_SUCCESS;
}
-TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *idx_val)
+/*
+ * The caller is to provide a correctly sized key, used only in
+ * the GUID index mode
+ */
+int ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val,
+ TDB_DATA *key)
{
- TDB_DATA key;
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct ldb_dn *dn;
if (ltdb->cache->GUID_index_attribute != NULL) {
- return ltdb_guid_to_key(module, ltdb, mem_ctx, idx_val);
+ return ltdb_guid_to_key(module, ltdb,
+ idx_val, key);
}
dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
if (dn == NULL) {
- errno = EINVAL;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ /*
+ * LDB_ERR_INVALID_DN_SYNTAX would just be confusing
+ * to the caller, as this in an invalid index value
+ */
+ return LDB_ERR_OPERATIONS_ERROR;
}
/* form the key */
- key = ltdb_key_dn(module, mem_ctx, dn);
+ *key = ltdb_key_dn(module, mem_ctx, dn);
TALLOC_FREE(dn);
- if (!key.dptr) {
- errno = ENOMEM;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ if (!key->dptr) {
+ return ldb_module_oom(module);
}
- return key;
+ return LDB_SUCCESS;
}
/*
@@ -283,6 +279,7 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA key;
const struct ldb_val *guid_val;
+ int ret;
if (ltdb->cache->GUID_index_attribute == NULL) {
return ltdb_key_dn(module, mem_ctx, msg->dn);
@@ -301,8 +298,25 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
return key;
}
- return ltdb_guid_to_key(module, ltdb, mem_ctx, guid_val);
+ /* In this case, allocate with talloc */
+ key.dptr = talloc_size(mem_ctx, LTDB_GUID_KEY_SIZE);
+ if (key.dptr == NULL) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ key.dsize = talloc_get_size(key.dptr);
+ ret = ltdb_guid_to_key(module, ltdb, guid_val, &key);
+
+ if (ret != LDB_SUCCESS) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ return key;
}
/*
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 0e6e9797d4b..cd77e3526ec 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -80,6 +80,7 @@ struct ltdb_context {
/* DB keys */
#define LTDB_GUID_KEY_PREFIX "GUID="
#define LTDB_GUID_SIZE 16
+#define LTDB_GUID_KEY_SIZE (LTDB_GUID_SIZE + sizeof(LTDB_GUID_KEY_PREFIX) - 1)
/* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c */
@@ -151,14 +152,15 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg);
-TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *guid_val);
-TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *idx_val);
+int ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_val *GUID_val,
+ TDB_DATA *key);
+int ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val,
+ TDB_DATA *key);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From 4a1448f4f98b860f0f912a50a03b888f37fd63ca Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 22:25:43 +1200
Subject: [PATCH 72/94] ldb_tdb: Clean up list in after use in
ltdb_key_dn_from_idx()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 78675e8ae56..765198bfd80 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -464,10 +464,12 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
ret = ltdb_index_dn_base_dn(module, ltdb, dn, list);
if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(list);
return ret;
}
if (list->count == 0) {
+ TALLOC_FREE(list);
return LDB_ERR_NO_SUCH_OBJECT;
}
if (list->count > 1) {
@@ -486,6 +488,7 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
/* The tdb_key memory is allocated by the caller */
ret = ltdb_guid_to_key(module, ltdb,
&list->dn[0], tdb_key);
+ TALLOC_FREE(list);
if (ret != LDB_SUCCESS) {
return LDB_ERR_OPERATIONS_ERROR;
--
2.11.0
From 37571717d8255530c9a297db5ea80b609a826f4d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:06:50 +1200
Subject: [PATCH 73/94] ldb_tdb: Avoid canonicalise and base64 work for DN
values, these are already OK
This is important with the GUID index, as a DN lookup is much more common now.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 100 ++++++++++++++++++++++++++++++++------------
1 file changed, 73 insertions(+), 27 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 765198bfd80..14ec207a3bd 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -733,45 +733,86 @@ int ltdb_index_transaction_cancel(struct ldb_module *module)
the caller is responsible for freeing
*/
static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
const char *attr, const struct ldb_val *value,
const struct ldb_schema_attribute **ap)
{
struct ldb_dn *ret;
struct ldb_val v;
- const struct ldb_schema_attribute *a;
- char *attr_folded;
+ const struct ldb_schema_attribute *a = NULL;
+ char *attr_folded = NULL;
+ const char *attr_for_dn = NULL;
int r;
+ bool should_b64_encode;
- attr_folded = ldb_attr_casefold(ldb, attr);
- if (!attr_folded) {
- return NULL;
+ if (attr[0] == '@') {
+ attr_for_dn = attr;
+ v = *value;
+ if (ap != NULL) {
+ *ap = NULL;
+ }
+ } else {
+ attr_folded = ldb_attr_casefold(ldb, attr);
+ if (!attr_folded) {
+ return NULL;
+ }
+
+ attr_for_dn = attr_folded;
+
+ a = ldb_schema_attribute_by_name(ldb, attr);
+ if (ap) {
+ *ap = a;
+ }
+ r = a->syntax->canonicalise_fn(ldb, ldb, value, &v);
+ if (r != LDB_SUCCESS) {
+ const char *errstr = ldb_errstring(ldb);
+ /* canonicalisation can be refused. For
+ example, a attribute that takes wildcards
+ will refuse to canonicalise if the value
+ contains a wildcard */
+ ldb_asprintf_errstring(ldb,
+ "Failed to create index "
+ "key for attribute '%s':%s%s%s",
+ attr, ldb_strerror(r),
+ (errstr?":":""),
+ (errstr?errstr:""));
+ talloc_free(attr_folded);
+ return NULL;
+ }
}
- a = ldb_schema_attribute_by_name(ldb, attr);
- if (ap) {
- *ap = a;
- }
- r = a->syntax->canonicalise_fn(ldb, ldb, value, &v);
- if (r != LDB_SUCCESS) {
- const char *errstr = ldb_errstring(ldb);
- /* canonicalisation can be refused. For example,
- a attribute that takes wildcards will refuse to canonicalise
- if the value contains a wildcard */
- ldb_asprintf_errstring(ldb, "Failed to create index key for attribute '%s':%s%s%s",
- attr, ldb_strerror(r), (errstr?":":""), (errstr?errstr:""));
- talloc_free(attr_folded);
- return NULL;
+ /*
+ * We do not base 64 encode a DN in a key, it has already been
+ * casefold and lineraized, that is good enough. That already
+ * avoids embedded NUL etc.
+ */
+ if (strcmp(attr, LTDB_IDXDN) == 0) {
+ should_b64_encode = false;
+
+ } else if (ltdb->cache->GUID_index_attribute != NULL
+ && strcmp(attr, LTDB_IDXONE) == 0) {
+ /*
+ * We can only change the behaviour for IDXONE when the GUID
+ * index is enabled
+ */
+ should_b64_encode = false;
+ } else {
+ should_b64_encode = ldb_should_b64_encode(ldb, &v);
}
- if (ldb_should_b64_encode(ldb, &v)) {
+
+ if (should_b64_encode) {
char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
if (!vstr) {
talloc_free(attr_folded);
return NULL;
}
- ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s::%s", LTDB_INDEX, attr_folded, vstr);
+ ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s::%s", LTDB_INDEX,
+ attr_for_dn, vstr);
talloc_free(vstr);
} else {
- ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s:%.*s", LTDB_INDEX, attr_folded, (int)v.length, (char *)v.data);
+ ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s:%.*s", LTDB_INDEX,
+ attr_for_dn,
+ (int)v.length, (char *)v.data);
}
if (v.data != value->data) {
@@ -870,7 +911,9 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
/* the attribute is indexed. Pull the list of DNs that match the
search criterion */
- dn = ltdb_index_key(ldb, tree->u.equality.attr, &tree->u.equality.value, NULL);
+ dn = ltdb_index_key(ldb, ltdb,
+ tree->u.equality.attr,
+ &tree->u.equality.value, NULL);
if (!dn) return LDB_ERR_OPERATIONS_ERROR;
ret = ltdb_dn_list_load(module, ltdb, dn, list);
@@ -1314,7 +1357,7 @@ static int ltdb_index_dn_attr(struct ldb_module *module,
/* work out the index key from the parent DN */
val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
val.length = strlen((char *)val.data);
- key = ltdb_index_key(ldb, attr, &val, NULL);
+ key = ltdb_index_key(ldb, ltdb, attr, &val, NULL);
if (!key) {
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1671,7 +1714,8 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn_key = ltdb_index_key(ldb, el->name, &el->values[v_idx], &a);
+ dn_key = ltdb_index_key(ldb, ltdb,
+ el->name, &el->values[v_idx], &a);
if (!dn_key) {
talloc_free(list);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1689,7 +1733,8 @@ static int ltdb_index_add1(struct ldb_module *module,
* DN -> GUID record
*/
if (list->count > 0 &&
- (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
+ ((a != NULL
+ && (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX)) ||
ldb_attr_cmp(el->name, LTDB_IDXDN) == 0)) {
/*
* We do not want to print info about a possibly
@@ -2020,7 +2065,8 @@ int ltdb_index_del_value(struct ldb_module *module,
return LDB_SUCCESS;
}
- dn_key = ltdb_index_key(ldb, el->name, &el->values[v_idx], NULL);
+ dn_key = ltdb_index_key(ldb, ltdb,
+ el->name, &el->values[v_idx], NULL);
if (!dn_key) {
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From 86ef5284c454c3a82361c947ba0476483bd3ff60 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:19:39 +1200
Subject: [PATCH 74/94] ldb_tdb: Give a good error message on add without an
objectGUID
(or whatever the @IDX_GUID value is)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index b4ce2470352..82ebe735729 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -292,6 +292,12 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
guid_val = ldb_msg_find_ldb_val(msg,
ltdb->cache->GUID_index_attribute);
if (guid_val == NULL) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Did not find GUID attribute %s "
+ "in %s, required for TDB record "
+ "key in " LTDB_IDXGUID " mode.",
+ ltdb->cache->GUID_index_attribute,
+ ldb_dn_get_linearized(msg->dn));
errno = EINVAL;
key.dptr = NULL;
key.dsize = 0;
--
2.11.0
From de4e9fd0c747286c421b2baa1d9803dc0c2d71ba Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 15:30:08 +1200
Subject: [PATCH 75/94] ldb_tdb: Describe index format and control points
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 113 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 14ec207a3bd..ca311a49cc3 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -149,6 +149,119 @@
* Author: Andrew Tridgell
*/
+/*
+
+LDB Index design and choice of TDB key:
+=======================================
+
+LDB has index records held as LDB objects with a special record like:
+
+dn: @INDEX:attr:value
+
+value may be base64 encoded, if it is deemed not printable:
+
+dn: @INDEX:attr::base64-value
+
+In each record, there is two possible formats:
+
+The original format is:
+-----------------------
+
+dn: @INDEX:NAME:DNSUPDATEPROXY
+ at IDXVERSION: 2
+ at IDX: CN=DnsUpdateProxy,CN=Users,DC=addom,DC=samba,DC=example,DC=com
+
+In this format, @IDX is multi-valued, one entry for each match
+
+The corrosponding entry is stored in a TDB record with key:
+
+DN=CN=DNSUPDATEPROXY,CN=USERS,DC=ADDOM,DC=SAMBA,DC=EXAMPLE,DC=COM
+
+(This allows a scope BASE search to directly find the record via
+a simple casefold of the DN).
+
+The original mixed-case DN is stored in the entry iself.
+
+
+The new 'GUID index' format is:
+-------------------------------
+
+dn: @INDEX:NAME:DNSUPDATEPROXY
+ at IDXVERSION: 3
+ at IDX: <binary GUID>[<binary GUID>[...]]
+
+The binary guid is 16 bytes, as bytes and not expanded as hexidecimal
+or pretty-printed. The GUID is chosen from the message to be stored
+by the @IDXGUID attribute on @INDEXLIST.
+
+If there are multiple values the @IDX value simply becomes longer,
+in multiples of 16.
+
+The corrosponding entry is stored in a TDB record with key:
+
+GUID=<binary GUID>
+
+This allows a very quick translation between the fixed-length index
+values and the TDB key, while seperating entries from other data
+in the TDB, should they be unlucky enough to start with the bytes of
+the 'DN=' prefix.
+
+Additionally, this allows a scope BASE search to directly find the
+record via a simple match on a GUID= extended DN, controlled via
+ at IDX_DN_GUID on @INDEXLIST
+
+Exception for special @ DNs:
+
+ at BASEINFO, @INDEXLIST and all other special DNs are stored as per the
+original format, as they are never referenced in an index and are used
+to bootstrap the database.
+
+
+Control points for choice of index mode
+---------------------------------------
+
+The choice of index and TDB key mode is made based (for example, from
+Samba) on entries in the @INDEXLIST DN:
+
+dn: @INDEXLIST
+ at IDXGUID: objectGUID
+ at IDX_DN_GUID: GUID
+
+By default, the original DN format is used.
+
+
+Control points for choosing indexed attributes
+----------------------------------------------
+
+ at IDXATTR controls if an attribute is indexed
+
+dn: @INDEXLIST
+ at IDXATTR: samAccountName
+ at IDXATTR: nETBIOSName
+
+
+C Override functions
+--------------------
+
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component)
+
+This is used, particularly in combination with the below, instead of
+the @IDXGUID and @IDX_DN_GUID values in @INDEXLIST.
+
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+ bool one_level_indexes);
+void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
+ ldb_attribute_handler_override_fn_t override,
+ void *private_data);
+
+When the above two functions are called in combination, the @INDEXLIST
+values are not read from the DB, so
+ldb_schema_set_override_GUID_index() must be called.
+
+*/
+
#include "ldb_tdb.h"
#include "ldb_private.h"
--
2.11.0
From 5527c85e1b7e0dc2353ac06a10212d6bed041cc4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 18:07:51 +1200
Subject: [PATCH 76/94] ldb_tdb: Clean up index records on ltdb_index_add_new()
failure.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index ca311a49cc3..068bedb1a20 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2144,10 +2144,29 @@ int ltdb_index_add_new(struct ldb_module *module,
ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
+ /*
+ * Because we can't trust the caller to be doing
+ * transactions properly, clean up any index for this
+ * entry rather than relying on a transaction
+ * cleanup
+ */
+
+ ltdb_index_delete(module, msg);
return ret;
}
- return ltdb_index_onelevel(module, msg, 1);
+ ret = ltdb_index_onelevel(module, msg, 1);
+ if (ret != LDB_SUCCESS) {
+ /*
+ * Because we can't trust the caller to be doing
+ * transactions properly, clean up any index for this
+ * entry rather than relying on a transaction
+ * cleanup
+ */
+ ltdb_index_delete(module, msg);
+ return ret;
+ }
+ return ret;
}
--
2.11.0
From 2b8038c7e515db35a12bb993d0807bfb5bcc9e27 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 10:02:16 +1200
Subject: [PATCH 77/94] ldb_tdb: Remove LTDB_FLAG_HIDDEN and ignore "HIDDEN" in
@ATTRIBUTES
This was (unintentionally) disabled by
6ef61825541131e16a03975cdbd344e2bbebf810 in 2006.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 25 +++++++++++--------------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 7bf5f156b45..5347a3be13b 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -36,7 +36,6 @@
#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
#define LTDB_FLAG_INTEGER (1<<1)
-#define LTDB_FLAG_HIDDEN (1<<2)
/* valid attribute flags */
static const struct {
@@ -45,7 +44,7 @@ static const struct {
} ltdb_valid_attr_flags[] = {
{ "CASE_INSENSITIVE", LTDB_FLAG_CASE_INSENSITIVE },
{ "INTEGER", LTDB_FLAG_INTEGER },
- { "HIDDEN", LTDB_FLAG_HIDDEN },
+ { "HIDDEN", 0 },
{ "NONE", 0 },
{ NULL, 0 }
};
@@ -150,7 +149,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
/* mapping these flags onto ldap 'syntaxes' isn't strictly correct,
but its close enough for now */
for (i=0;i<attrs_msg->num_elements;i++) {
- unsigned flags;
+ unsigned flags, attr_flags;
const char *syntax;
const struct ldb_schema_syntax *s;
const struct ldb_schema_attribute *a =
@@ -167,17 +166,15 @@ static int ltdb_attributes_load(struct ldb_module *module)
attrs_msg->elements[i].name);
goto failed;
}
- switch (flags & ~LTDB_FLAG_HIDDEN) {
- case 0:
- syntax = LDB_SYNTAX_OCTET_STRING;
- break;
- case LTDB_FLAG_CASE_INSENSITIVE:
+
+ /* These are not currently flags, each is exclusive */
+ if (flags == LTDB_FLAG_CASE_INSENSITIVE) {
syntax = LDB_SYNTAX_DIRECTORY_STRING;
- break;
- case LTDB_FLAG_INTEGER:
+ } else if (flags == LTDB_FLAG_INTEGER) {
syntax = LDB_SYNTAX_INTEGER;
- break;
- default:
+ } else if (flags == 0) {
+ syntax = LDB_SYNTAX_OCTET_STRING;
+ } else {
ldb_debug(ldb, LDB_DEBUG_ERROR,
"Invalid flag combination 0x%x for '%s' "
"in @ATTRIBUTES",
@@ -194,12 +191,12 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
- flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
+ attr_flags = LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
r = ldb_schema_attribute_fill_with_syntax(ldb,
attrs,
attrs_msg->elements[i].name,
- flags, s,
+ attr_flags, s,
&attrs[num_loaded_attrs + ldb->schema.num_attributes]);
if (r != 0) {
goto failed;
--
2.11.0
From 9391a79dab70be6d23c7c3681e7640972d6ec5c2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 12:36:43 +1200
Subject: [PATCH 78/94] ldb_tdb: Add UNIQUE_INDEX as a possible flag
This allows easy testing of our unique index code and behaivour from python
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 5347a3be13b..5b90bd99f47 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -36,6 +36,7 @@
#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
#define LTDB_FLAG_INTEGER (1<<1)
+#define LTDB_FLAG_UNIQUE_INDEX (1<<2)
/* valid attribute flags */
static const struct {
@@ -45,6 +46,7 @@ static const struct {
{ "CASE_INSENSITIVE", LTDB_FLAG_CASE_INSENSITIVE },
{ "INTEGER", LTDB_FLAG_INTEGER },
{ "HIDDEN", 0 },
+ { "UNIQUE_INDEX", LTDB_FLAG_UNIQUE_INDEX},
{ "NONE", 0 },
{ NULL, 0 }
};
@@ -149,7 +151,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
/* mapping these flags onto ldap 'syntaxes' isn't strictly correct,
but its close enough for now */
for (i=0;i<attrs_msg->num_elements;i++) {
- unsigned flags, attr_flags;
+ unsigned flags = 0, attr_flags = 0;
const char *syntax;
const struct ldb_schema_syntax *s;
const struct ldb_schema_attribute *a =
@@ -167,6 +169,11 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
+ if (flags & LTDB_FLAG_UNIQUE_INDEX) {
+ attr_flags = LDB_ATTR_FLAG_UNIQUE_INDEX;
+ }
+ flags &= ~LTDB_FLAG_UNIQUE_INDEX;
+
/* These are not currently flags, each is exclusive */
if (flags == LTDB_FLAG_CASE_INSENSITIVE) {
syntax = LDB_SYNTAX_DIRECTORY_STRING;
@@ -191,7 +198,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
- attr_flags = LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
+ attr_flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
r = ldb_schema_attribute_fill_with_syntax(ldb,
attrs,
--
2.11.0
From 8849e7178a13da094e4c966e93954233dcab5d1b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 11:28:23 +1200
Subject: [PATCH 79/94] ldb: Add more tests covering in-transaction GUID index
and unique index behaviour
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++++++++++++++---
1 file changed, 173 insertions(+), 12 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 4c35e44ee28..b2e31c23f4e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -984,6 +984,8 @@ class AddModifyTests(TestCase):
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
+ self.l.add({"dn": "@ATTRIBUTES",
+ "objectUUID": "UNIQUE_INDEX"})
def test_add_dup(self):
self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
@@ -1023,6 +1025,88 @@ class AddModifyTests(TestCase):
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde2"})
+ def test_add_move_fail_move_move(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on duplicate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ self.l.rename("OU=DUP2,DC=SAMBA,DC=ORG",
+ "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+
+ res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde1)")
+ self.assertEqual(len(res2), 1)
+ self.assertEqual(str(res2[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
+ res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde2)")
+ self.assertEqual(len(res3), 1)
+ self.assertEqual(str(res3[0].dn), "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ def test_move_missing(self):
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on missing")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_NO_SUCH_OBJECT)
+
+ def test_move_missing2(self):
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on missing")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_NO_SUCH_OBJECT)
+
+ def test_move_fail_move_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on duplicate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ self.l.rename("OU=DUP2,DC=SAMBA,DC=ORG",
+ "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde3"})
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
@@ -1030,20 +1114,9 @@ class IndexedAddModifyTests(AddModifyTests):
def setUp(self):
super(IndexedAddModifyTests, self).setUp()
self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
"@IDXONE": [b"1"]})
-class GUIDIndexedAddModifyTests(AddModifyTests):
- """Test searches using the index, to ensure the index doesn't
- break things"""
- def setUp(self):
- super(GUIDIndexedAddModifyTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
-
def test_duplicate_GUID(self):
try:
self.l.add({"dn": "OU=DUPGUID,DC=SAMBA,DC=ORG",
@@ -1055,6 +1128,94 @@ class GUIDIndexedAddModifyTests(AddModifyTests):
enum = err.args[0]
self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+ def test_duplicate_name_dup_GUID(self):
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"a123456789abcdef"})
+ try:
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"a123456789abcdef"})
+ self.fail("Should have failed adding dupliate GUID")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ def test_duplicate_name_dup_GUID2(self):
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"abc3456789abcdef"})
+ try:
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"aaa3456789abcdef"})
+ self.fail("Should have failed adding dupliate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ # Checking the GUID didn't stick in the index
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"aaa3456789abcdef"})
+
+ def test_add_dup_guid_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ try:
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.fail("Should have failed on duplicate GUID")
+
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedAddModifyTests, self).setUp()
+ indexlist = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
+ m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
+ self.l.modify(m)
+
+
+class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
+ """Test GUID index behaviour insdie the transaction"""
+ def setUp(self):
+ super(GUIDTransIndexedAddModifyTests, self).setUp()
+ self.l.transaction_start()
+
+ def tearDown(self):
+ self.l.transaction_commit()
+
+class TransIndexedAddModifyTests(IndexedAddModifyTests):
+ """Test index behaviour insdie the transaction"""
+ def setUp(self):
+ super(TransIndexedAddModifyTests, self).setUp()
+ self.l.transaction_start()
+
+ def tearDown(self):
+ self.l.transaction_commit()
class DnTests(TestCase):
--
2.11.0
From 3104b18025bd3ce578932120e9c0f88aac345ed6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 11:49:02 +1200
Subject: [PATCH 80/94] ldb_tdb: Remove incorrect early return from re-index
The ltdb->cache->attribute_indexes test is not correct with the GUID index mode
so for consistency remove it. This will make re-index on a large un-indexed
database slower, but that is better than making the wrong choice on a large
GUID-indexed database.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 5 -----
1 file changed, 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 068bedb1a20..bf508e8c37a 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2594,11 +2594,6 @@ int ltdb_reindex(struct ldb_module *module)
return LDB_ERR_OPERATIONS_ERROR;
}
- /* if we don't have indexes we have nothing todo */
- if (!ltdb->cache->attribute_indexes) {
- return LDB_SUCCESS;
- }
-
ctx.module = module;
ctx.error = 0;
--
2.11.0
From 28624b383f939761fbaee30a0951697066bd9948 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 13:16:31 +1200
Subject: [PATCH 81/94] ldb_tdb: Print progress messages on re-index
A re-index of 10,000 entries is slow enough and rare enought that we can
justify the message being at LDB_DEBUG_WARNING as otherwise the administrator
will be sure the "lockup" was one.
The default for ldb is to print LDB_DEBUG_WARNING in comand-line tools
and the default for Samba is to log it at level 2.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index bf508e8c37a..1008bac9b03 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2370,6 +2370,7 @@ static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, vo
struct ltdb_reindex_context {
struct ldb_module *module;
int error;
+ uint32_t count;
};
/*
@@ -2474,6 +2475,13 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
talloc_free(msg);
+ ctx->count++;
+ if (ctx->count % 10000 == 0) {
+ ldb_debug(ldb, LDB_DEBUG_WARNING,
+ "Reindexing: re-keyed %u records so far",
+ ctx->count);
+ }
+
return 0;
}
@@ -2555,6 +2563,13 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
talloc_free(msg);
+ ctx->count++;
+ if (ctx->count % 10000 == 0) {
+ ldb_debug(ldb, LDB_DEBUG_WARNING,
+ "Reindexing: re-indexed %u records so far",
+ ctx->count);
+ }
+
return 0;
}
@@ -2596,6 +2611,7 @@ int ltdb_reindex(struct ldb_module *module)
ctx.module = module;
ctx.error = 0;
+ ctx.count = 0;
/* now traverse adding any indexes for normal LDB records */
ret = tdb_traverse(ltdb->tdb, re_key, &ctx);
@@ -2613,6 +2629,7 @@ int ltdb_reindex(struct ldb_module *module)
}
ctx.error = 0;
+ ctx.count = 0;
/* now traverse adding any indexes for normal LDB records */
ret = tdb_traverse(ltdb->tdb, re_index, &ctx);
@@ -2629,5 +2646,11 @@ int ltdb_reindex(struct ldb_module *module)
return ctx.error;
}
+ if (ctx.count > 10000) {
+ ldb_debug(ldb_module_get_ctx(module),
+ LDB_DEBUG_WARNING, "Reindexing: re_index successful on %s, "
+ "final index write-out will be in transaction commit",
+ tdb_name(ltdb->tdb));
+ }
return LDB_SUCCESS;
}
--
2.11.0
From fb3b08439a1ee522e40e945b9eded86cf69e738e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 22:00:40 +1200
Subject: [PATCH 82/94] ldb_tdb: Update comments for the delete_index() pass of
ltdb_reindex()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 1008bac9b03..27bc605944e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2330,7 +2330,12 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
/*
- traversal function that deletes all @INDEX records
+ traversal function that deletes all @INDEX records in the in-memory
+ TDB.
+
+ This does not touch the actual DB, that is done at transaction
+ commit, which in turn greatly reduces DB churn as we will likely
+ be able to do a direct update into the old record.
*/
static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state)
{
@@ -2355,6 +2360,11 @@ static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, vo
v.length = strnlen((char *)key.dptr, key.dsize) - 3;
dn = ldb_dn_from_ldb_val(ltdb, ldb_module_get_ctx(module), &v);
+
+ /*
+ * This does not actually touch the DB quite yet, just
+ * the in-memory index cache
+ */
ret = ltdb_dn_list_store(module, dn, &list);
if (ret != LDB_SUCCESS) {
ldb_asprintf_errstring(ldb_module_get_ctx(module),
--
2.11.0
From 461e902683a7e11d1e482cebb7c6efd8bf07ab35 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 13:00:08 +1200
Subject: [PATCH 83/94] ldb_tdb: Also ban a (indexed) search against like
@IDXDN=foo
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 27bc605944e..3bfae1f970f 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1056,6 +1056,12 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
list->count = 0;
return LDB_SUCCESS;
}
+ if (tree->u.equality.attr[0] == '@') {
+ /* Do not allow a indexed search against an @ */
+ list->dn = NULL;
+ list->count = 0;
+ return LDB_SUCCESS;
+ }
if (ldb_attr_dn(tree->u.equality.attr) == 0) {
if (ltdb->cache->GUID_index_attribute == NULL) {
list->dn = talloc_array(list, struct ldb_val, 1);
--
2.11.0
From aec7048123cb950a9d75501d78e874924cdec961 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:54:00 +1200
Subject: [PATCH 84/94] TODO: Release ldb 1.3.0
* GUID Index support.
NOTE: When activated by setting @IDXGUID in the @INDEXLIST dn, all
entries in the DB are re-keyed in a way that is NOT visible to
ldb 1.2.2 and earlier. To re-key back to the previous format, remove
the @IDXGUID attribute from @INDEXLIST using ldb 1.2.2 or later.
(ldb 1.2.2 can re-key, but not otherwise read, the new DB format).
* Give LDB_ERR_CONSTRAINT_VIOLATION, not LDB_ERR_ENTRY_ALREADY_EXISTS
when a duplicate value is detected in a unique index
* Print status information during a > 10,000 entry re-index
(as this can be slow)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ABI/ldb-1.3.0.sigs | 279 ++++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.3.0.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 284 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.3.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.3.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
diff --git a/lib/ldb/ABI/ldb-1.3.0.sigs b/lib/ldb/ABI/ldb-1.3.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.3.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.3.0.sigs b/lib/ldb/ABI/pyldb-util-1.3.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.3.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index bd17b7b9edc..de7eef72c14 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.2.2'
+VERSION = '1.3.0'
blddir = 'bin'
--
2.11.0
From e760daf681b323378711615849af159f6be01d3d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 17:37:05 +1200
Subject: [PATCH 85/94] selftest: sort dbcheck output to avoid sort order
impacting results
The GUID index code will change the returned results order
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
testprogs/blackbox/dbcheck-links.sh | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh
index fb66d14e6f1..17973e97429 100755
--- a/testprogs/blackbox/dbcheck-links.sh
+++ b/testprogs/blackbox/dbcheck-links.sh
@@ -59,7 +59,9 @@ dbcheck() {
if [ "$?" != "1" ]; then
return 1
fi
- diff $tmpfile $release_dir/expected-dbcheck-link-output.txt
+ sort $tmpfile > $tmpfile.sorted
+ sort $release_dir/expected-dbcheck-link-output.txt > $tmpfile.expected
+ diff -u $tmpfile.sorted $tmpfile.expected
if [ "$?" != "0" ]; then
return 1
fi
--
2.11.0
From 7dc259075b9264c5c4d7637db57f4ceac86585e8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:21:29 +1200
Subject: [PATCH 86/94] provision: make clear that the tmp ldb is running in
@IDXGUID mode
This happended when the schema was set on the DB, forcing the full set of Samba behaviours
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/schema.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/python/samba/schema.py b/python/samba/schema.py
index 839d681825a..17a171960b9 100644
--- a/python/samba/schema.py
+++ b/python/samba/schema.py
@@ -126,12 +126,15 @@ class Schema(object):
self.ldb.connect(url=schemadb_path)
self.ldb.transaction_start()
try:
+ # These are actually ignored, as the schema has been forced
+ # when the ldb object was created, and that overrides this
self.ldb.add_ldif("""dn: @ATTRIBUTES
linkID: INTEGER
dn: @INDEXLIST
@IDXATTR: linkID
@IDXATTR: attributeSyntax
+ at IDXGUID: objectGUID
""")
# These bits of LDIF are supplied when the Schema object is created
self.ldb.add_ldif(self.schema_dn_add)
--
2.11.0
From e57ac6381987f55f6d026db35c04bb00c973d0e4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:31:21 +1200
Subject: [PATCH 87/94] provision: Add a fixed objectGUID to the tmp DB used
for LDAP backend schema work
This DB holds a copy of the schema, but now needs to have an objectGUID on each record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/schema.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/python/samba/schema.py b/python/samba/schema.py
index 17a171960b9..3828003f4cf 100644
--- a/python/samba/schema.py
+++ b/python/samba/schema.py
@@ -136,8 +136,12 @@ dn: @INDEXLIST
@IDXATTR: attributeSyntax
@IDXGUID: objectGUID
""")
+
+ schema_dn_add = self.schema_dn_add \
+ + "objectGUID: 24e2ca70-b093-4ae8-84c0-2d7ac652a1b8\n"
+
# These bits of LDIF are supplied when the Schema object is created
- self.ldb.add_ldif(self.schema_dn_add)
+ self.ldb.add_ldif(schema_dn_add)
self.ldb.modify_ldif(self.schema_dn_modify)
self.ldb.add_ldif(self.schema_data)
except:
--
2.11.0
From db1b3aabafcc62a7315d6c162d4e48b06669093d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:33:19 +1200
Subject: [PATCH 88/94] provision: Add a fixed GUID to the samba4top
objectclass definition
This is only used in the OpenLDAP backend and will certainly be removed before this becomes production.
(a production backend will use the real AD top objectclass)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/setup/schema_samba4.ldif | 1 +
1 file changed, 1 insertion(+)
diff --git a/source4/setup/schema_samba4.ldif b/source4/setup/schema_samba4.ldif
index fcfaf9819b7..4fd729e3149 100644
--- a/source4/setup/schema_samba4.ldif
+++ b/source4/setup/schema_samba4.ldif
@@ -293,6 +293,7 @@ objectClass: top
objectClass: classSchema
cn: Samba4Top
subClassOf: top
+objectGUID: 4af54ff0-ff3c-4f17-8fb0-611ec83ddfb4
governsID: 1.3.6.1.4.1.7165.4.2.1
mayContain: msDS-ObjectReferenceBL
rDNAttID: cn
--
2.11.0
From 1cb616fd4fd8aa32fb460ce66134c80b8493009b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:58:57 +1200
Subject: [PATCH 89/94] dsdb: Set that Samba uses the GUID index in LDB
This is optional, but only to aid the downgrade script (and in case
there is some major issue found with it). We don't support that mode,
as that would require us to test and maintain multiple code paths and
not optimise queries to be GUID centric.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/schema/schema_set.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 8141e323e50..23d881f0bdb 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -69,9 +69,16 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
struct ldb_message *msg;
struct ldb_message *msg_idx;
+ struct loadparm_context *lp_ctx =
+ talloc_get_type(ldb_get_opaque(ldb, "loadparm"),
+ struct loadparm_context);
/* setup our own attribute name to schema handler */
ldb_schema_attribute_set_override_handler(ldb, dsdb_attribute_handler_override, schema);
ldb_schema_set_override_indexlist(ldb, true);
+ if (lp_ctx == NULL ||
+ lpcfg_parm_bool(lp_ctx, NULL, "dsdb", "guid index", true)) {
+ ldb_schema_set_override_GUID_index(ldb, "objectGUID", "GUID");
+ }
if (!write_indices_and_attributes) {
return ret;
@@ -108,6 +115,18 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
goto op_error;
}
+ if (lp_ctx == NULL ||
+ lpcfg_parm_bool(lp_ctx, NULL, "dsdb", "guid index", true)) {
+ ret = ldb_msg_add_string(msg_idx, "@IDXGUID", "objectGUID");
+ if (ret != LDB_SUCCESS) {
+ goto op_error;
+ }
+
+ ret = ldb_msg_add_string(msg_idx, "@IDX_DN_GUID", "GUID");
+ if (ret != LDB_SUCCESS) {
+ goto op_error;
+ }
+ }
ret = ldb_msg_add_string(msg_idx, "@IDXVERSION", SAMDB_INDEXING_VERSION);
if (ret != LDB_SUCCESS) {
--
2.11.0
From 93b02d0bb74036a866de23ae847702a2b7706393 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 15:31:55 +1200
Subject: [PATCH 90/94] Do not re-use the attribute @IDXVERSION for
SAMDB_INDEXING_VERSION
Confusing these two concepts is not a good idea, SAMDB_INDEXING_VERSION refers to
a change in a Samba rule to canonicalise one of our attributes, not the
in-DB index format.
As we already change @INDEXLIST in this version, this commit
is at no extra cost.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/schema/schema_set.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 23d881f0bdb..ca7a307212a 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -128,7 +128,7 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
}
}
- ret = ldb_msg_add_string(msg_idx, "@IDXVERSION", SAMDB_INDEXING_VERSION);
+ ret = ldb_msg_add_string(msg_idx, "@SAMDB_INDEXING_VERSION", SAMDB_INDEXING_VERSION);
if (ret != LDB_SUCCESS) {
goto op_error;
}
--
2.11.0
From 50b68d05a2981f5c3c139b37fc60b6862d4facc8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 13:53:19 +1200
Subject: [PATCH 91/94] dsdb: Only trigger a re-index once per @INDEXLIST
modification
A modify of both @INDEXLIST and @ATTRIBUTES will still trigger two re-index passes
but that is a task for later.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=9527
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/samdb/ldb_modules/partition.c | 90 +++++++++++++++++++++++++-----
1 file changed, 75 insertions(+), 15 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c
index c304efa645d..426fce36e4a 100644
--- a/source4/dsdb/samdb/ldb_modules/partition.c
+++ b/source4/dsdb/samdb/ldb_modules/partition.c
@@ -432,30 +432,90 @@ static int partition_copy_all(struct ldb_module *module,
return search_ret;
}
- /* now delete the object in the other partitions. Once that is
- done we will re-add the object, if search_ret was not
- LDB_ERR_NO_SUCH_OBJECT
+ /* now delete the object in the other partitions, if requried
*/
+ if (search_ret == LDB_ERR_NO_SUCH_OBJECT) {
+ for (i=0; data->partitions && data->partitions[i]; i++) {
+ int pret;
+ pret = dsdb_module_del(data->partitions[i]->module,
+ dn,
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+ if (pret != LDB_SUCCESS && pret != LDB_ERR_NO_SUCH_OBJECT) {
+ /* we should only get success or no
+ such object from the other partitions */
+ return pret;
+ }
+ }
+
+ return ldb_module_done(req, NULL, NULL, LDB_SUCCESS);
+ }
+
+ /* now add/modify in the other partitions */
for (i=0; data->partitions && data->partitions[i]; i++) {
+ struct ldb_message *modify_msg = NULL;
int pret;
- pret = dsdb_module_del(data->partitions[i]->module, dn, DSDB_FLAG_NEXT_MODULE, req);
- if (pret != LDB_SUCCESS && pret != LDB_ERR_NO_SUCH_OBJECT) {
- /* we should only get success or no
- such object from the other partitions */
+ unsigned int el_idx;
+
+ pret = dsdb_module_add(data->partitions[i]->module,
+ res->msgs[0],
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+ if (pret == LDB_SUCCESS) {
+ continue;
+ }
+
+ if (pret != LDB_ERR_ENTRY_ALREADY_EXISTS) {
return pret;
}
- }
+ modify_msg = ldb_msg_copy(req, res->msgs[0]);
+ if (modify_msg == NULL) {
+ return ldb_module_oom(module);
+ }
- if (search_ret != LDB_ERR_NO_SUCH_OBJECT) {
- /* now re-add in the other partitions */
- for (i=0; data->partitions && data->partitions[i]; i++) {
- int pret;
- pret = dsdb_module_add(data->partitions[i]->module, res->msgs[0], DSDB_FLAG_NEXT_MODULE, req);
- if (pret != LDB_SUCCESS) {
- return pret;
+ /*
+ * mark all the message elements as
+ * LDB_FLAG_MOD_REPLACE
+ */
+ for (el_idx=0;
+ el_idx < modify_msg->num_elements;
+ el_idx++) {
+ modify_msg->elements[el_idx].flags
+ = LDB_FLAG_MOD_REPLACE;
+ }
+
+ if (req->operation == LDB_MODIFY) {
+ const struct ldb_message *req_msg = req->op.mod.message;
+ /*
+ * mark elements to be removed, if there were
+ * deleted entirely above we need to delete
+ * them here too
+ */
+ for (el_idx=0; el_idx < req_msg->num_elements; el_idx++) {
+ if (req_msg->elements[el_idx].flags & LDB_FLAG_MOD_DELETE
+ || ((req_msg->elements[el_idx].flags & LDB_FLAG_MOD_REPLACE) &&
+ req_msg->elements[el_idx].num_values == 0)) {
+ if (ldb_msg_find_element(modify_msg,
+ req_msg->elements[el_idx].name) != NULL) {
+ continue;
+ }
+ ldb_msg_add_empty(modify_msg,
+ req_msg->elements[el_idx].name,
+ LDB_FLAG_MOD_REPLACE,
+ NULL);
+ }
}
}
+
+ pret = dsdb_module_modify(data->partitions[i]->module,
+ modify_msg,
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+
+ if (pret != LDB_SUCCESS) {
+ return pret;
+ }
}
return ldb_module_done(req, NULL, NULL, LDB_SUCCESS);
--
2.11.0
From 2893107ab2a5f7d003898d0d847b6309194e8797 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 21:39:44 +1200
Subject: [PATCH 92/94] scripting: Add script (backportable) to undo a GUID
index
This script allows the DB to be read, and re-indexed, by an earlier Samba version,
most likely 4.7 with some backported patches.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/scripting/bin/sambaundoguididx | 84 ++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
create mode 100755 source4/scripting/bin/sambaundoguididx
diff --git a/source4/scripting/bin/sambaundoguididx b/source4/scripting/bin/sambaundoguididx
new file mode 100755
index 00000000000..24a95e20d7f
--- /dev/null
+++ b/source4/scripting/bin/sambaundoguididx
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+import optparse
+import sys
+
+# Find right directory when running from source tree
+sys.path.insert(0, "bin/python")
+
+
+import samba
+import ldb
+import urllib
+import os
+from samba import getopt as options
+from samba.samdb import SamDB
+from samba.dbchecker import dbcheck
+from samba.credentials import Credentials
+parser = optparse.OptionParser("sambaundoguididx")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(options.VersionOptions(parser))
+parser.add_option("-H", "--URL", help="LDB URL for database",
+ type=str, metavar="URL", dest="H")
+opts, args = parser.parse_args()
+
+if len(args) != 0:
+ parser.print_usage()
+ sys.exit(1)
+
+lp_ctx = sambaopts.get_loadparm()
+lp_ctx.set("dsdb:guid index", "false")
+
+if opts.H is None:
+ url = lp_ctx.samdb_url()
+else:
+ url = opts.H
+
+samdb = ldb.Ldb(url=url, options=["modules:"])
+
+partitions = samdb.search(base="@PARTITION",
+ scope=ldb.SCOPE_BASE,
+ attrs=["partition"])
+
+modmsg = ldb.Message()
+modmsg.dn = ldb.Dn(samdb, '@INDEXLIST')
+modmsg.add(ldb.MessageElement(
+ elements=[],
+ flags=ldb.FLAG_MOD_REPLACE,
+ name='@IDXGUID'))
+modmsg.add(ldb.MessageElement(
+ elements=[],
+ flags=ldb.FLAG_MOD_REPLACE,
+ name='@IDX_DN_GUID'))
+
+samdb.transaction_start()
+samdb.modify(modmsg)
+
+privatedir = os.path.dirname(url)
+
+dbs = []
+for part in partitions[0]['partition']:
+ file_quoted = part.split(":")[1]
+ tdbname = urllib.unquote(file_quoted)
+ tdbpath = os.path.join(privatedir, tdbname)
+
+ db = ldb.Ldb(url=tdbpath, options=["modules:"])
+ db.transaction_start()
+ db.modify(modmsg)
+ dbs.append(db)
+
+for db in dbs:
+ db.transaction_commit()
+
+samdb.transaction_commit()
+
+print "Re-opening with the full DB stack"
+samdb = SamDB(url=url,
+ lp=lp_ctx)
+print "Re-triggering another re-index"
+chk = dbcheck(samdb)
+
+chk.reindex_database()
+
+print "Your database has been downgraded to DN-based index values."
+
+print "NOTE: Any use of a Samba 4.8 tool including ldbsearch will auto-upgrade back to GUID index mode"
--
2.11.0
From d67d6ee7b59d75be3cfbdcb86a11c508016b00d8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 15:49:43 +1200
Subject: [PATCH 93/94] ldb: Add test showing a search against the index is not
possible
This is not actually a great test, as the filter would
fail to match these anyway, but it at least checks the
codepath is safe.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index b2e31c23f4e..7189e97c303 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -950,6 +950,22 @@ class SearchTests(TestCase):
expression="(&(ou=ouX)(y=a))")
self.assertEqual(len(res11), 0)
+ def test_subtree_and_idx_record(self):
+ """Testing a search against the index record"""
+
+ res11 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(@IDXDN=DC=SAMBA,DC=ORG)")
+ self.assertEqual(len(res11), 0)
+
+ def test_subtree_and_idxone_record(self):
+ """Testing a search against the index record"""
+
+ res11 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(@IDXONE=DC=SAMBA,DC=ORG)")
+ self.assertEqual(len(res11), 0)
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
--
2.11.0
From 499aa6c2a519abcb39f2219e57b60990a2bb6293 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 00:01:15 +1200
Subject: [PATCH 94/94] HACK: mark dnscmd tests as flapping
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
selftest/flapping.d/dns-wildcard | 2 ++
1 file changed, 2 insertions(+)
create mode 100644 selftest/flapping.d/dns-wildcard
diff --git a/selftest/flapping.d/dns-wildcard b/selftest/flapping.d/dns-wildcard
new file mode 100644
index 00000000000..2075a999232
--- /dev/null
+++ b/selftest/flapping.d/dns-wildcard
@@ -0,0 +1,2 @@
+#DNS wildcard tests broken by correct escaping of the ldap filter
+^samba.tests.samba_tool.dnscmd
--
2.11.0
More information about the samba-technical
mailing list