[PATCH] GUID index for LDB
Andrew Bartlett
abartlet at samba.org
Tue Sep 19 05:14:50 UTC 2017
On Wed, 2017-09-13 at 17:16 +1200, Andrew Bartlett via samba-technical
wrote:
> On Tue, 2017-09-12 at 23:10 +1200, Andrew Bartlett via samba-
> technical
> wrote:
> > On Tue, 2017-09-12 at 13:21 +1200, Andrew Bartlett via samba-
> > technical
> > wrote:
> > >
> > > In my current branch I've added some incremental output during
> > > the
> > > re-
> > > index.
> >
> > Attached is my current branch. A recent version of branch passed a
> > full autobuild (multiple times), and I trust this will also.
> >
> > Please review, as the only outstanding things as far as I know are:
> > - any required tests for the downgrade script
> > - the dns wildcard issue
>
> Attached is the current patch set. Please review so I can get this
> in
> to master soon.
Garming has given me a review on this patch set. To ease backporting
I'll try and land the ldb read-only set Gary sent first, with an
earlier LDB version number.
Thanks,
Andrew Bartlett
--
Andrew Bartlett
https://samba.org/~abartlet/
Authentication Developer, Samba Team https://samba.org
Samba Development and Support, Catalyst IT
https://catalyst.net.nz/services/samba
-------------- next part --------------
From ac9c3fef2e0fc4a0546ac2e526c3e59b577da388 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 11:48:46 +1200
Subject: [PATCH 01/88] s4-dnsserver: Always encode user-supplied names when
looking up DNS records
BUG: https://bugzilla.samba.org/show_bug.cgi?id=12994
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/rpc_server/dnsserver/dcerpc_dnsserver.c | 15 ++++++++++++---
source4/rpc_server/dnsserver/dnsdb.c | 15 +++++++++++----
2 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/source4/rpc_server/dnsserver/dcerpc_dnsserver.c b/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
index 286da18346b..120d4b9a3d7 100644
--- a/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
+++ b/source4/rpc_server/dnsserver/dcerpc_dnsserver.c
@@ -1674,10 +1674,13 @@ static WERROR dnsserver_enumerate_root_records(struct dnsserver_state *dsstate,
/* Add any additional records */
if (select_flag & DNS_RPC_VIEW_ADDITIONAL_DATA) {
for (i=0; i<add_count; i++) {
+ char *encoded_name
+ = ldb_binary_encode_string(tmp_ctx,
+ add_names[i]);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))",
- add_names[i]);
+ encoded_name);
if (ret != LDB_SUCCESS || res->count == 0) {
talloc_free(res);
continue;
@@ -1744,10 +1747,12 @@ static WERROR dnsserver_enumerate_records(struct dnsserver_state *dsstate,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(!(dNSTombstoned=TRUE)))");
} else {
+ char *encoded_name
+ = ldb_binary_encode_string(tmp_ctx, name);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(|(name=%s)(name=*.%s))(!(dNSTombstoned=TRUE)))",
- name, name);
+ encoded_name, encoded_name);
}
if (ret != LDB_SUCCESS) {
talloc_free(tmp_ctx);
@@ -1818,11 +1823,15 @@ static WERROR dnsserver_enumerate_records(struct dnsserver_state *dsstate,
/* Search all the available zones for additional name */
for (z2 = dsstate->zones; z2; z2 = z2->next) {
+ char *encoded_name;
name = dns_split_node_name(tmp_ctx, add_names[i], z2->name);
+ encoded_name
+ = ldb_binary_encode_string(tmp_ctx,
+ name);
ret = ldb_search(dsstate->samdb, tmp_ctx, &res, z2->zone_dn,
LDB_SCOPE_ONELEVEL, attrs,
"(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))",
- name);
+ encoded_name);
talloc_free(name);
if (ret != LDB_SUCCESS) {
continue;
diff --git a/source4/rpc_server/dnsserver/dnsdb.c b/source4/rpc_server/dnsserver/dnsdb.c
index da37878ce2c..15fe5657d3a 100644
--- a/source4/rpc_server/dnsserver/dnsdb.c
+++ b/source4/rpc_server/dnsserver/dnsdb.c
@@ -364,10 +364,12 @@ WERROR dnsserver_db_add_empty_node(TALLOC_CTX *mem_ctx,
const char * const attrs[] = { "name", NULL };
struct ldb_result *res;
struct ldb_dn *dn;
+ char *encoded_name = ldb_binary_encode_string(mem_ctx, name);
int ret;
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_BASE, attrs,
- "(&(objectClass=dnsNode)(name=%s))", name);
+ "(&(objectClass=dnsNode)(name=%s))",
+ encoded_name);
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
@@ -406,6 +408,7 @@ WERROR dnsserver_db_add_record(TALLOC_CTX *mem_ctx,
int serial;
WERROR werr;
bool was_tombstoned = false;
+ char *encoded_name = ldb_binary_encode_string(mem_ctx, name);
werr = dns_to_dnsp_convert(mem_ctx, add_record, &rec, true);
if (!W_ERROR_IS_OK(werr)) {
@@ -436,7 +439,8 @@ WERROR dnsserver_db_add_record(TALLOC_CTX *mem_ctx,
rec->dwTimeStamp = t;
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_ONELEVEL, attrs,
- "(&(objectClass=dnsNode)(name=%s))", name);
+ "(&(objectClass=dnsNode)(name=%s))",
+ encoded_name);
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
@@ -524,6 +528,7 @@ WERROR dnsserver_db_update_record(TALLOC_CTX *mem_ctx,
int ret, i;
int serial;
WERROR werr;
+ char *encoded_name = ldb_binary_encode_string(mem_ctx, name);
werr = dns_to_dnsp_convert(mem_ctx, add_record, &arec, true);
if (!W_ERROR_IS_OK(werr)) {
@@ -541,7 +546,8 @@ WERROR dnsserver_db_update_record(TALLOC_CTX *mem_ctx,
arec->dwTimeStamp = t;
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_ONELEVEL, attrs,
- "(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))", name);
+ "(&(objectClass=dnsNode)(name=%s)(!(dNSTombstoned=TRUE)))",
+ encoded_name);
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
@@ -642,7 +648,8 @@ WERROR dnsserver_db_delete_record(TALLOC_CTX *mem_ctx,
}
ret = ldb_search(samdb, mem_ctx, &res, z->zone_dn, LDB_SCOPE_ONELEVEL, attrs,
- "(&(objectClass=dnsNode)(name=%s))", name);
+ "(&(objectClass=dnsNode)(name=%s))",
+ ldb_binary_encode_string(mem_ctx, name));
if (ret != LDB_SUCCESS) {
return WERR_INTERNAL_DB_ERROR;
}
--
2.11.0
From 204ae175520395d11a8a91a820ba6509a3bc65e4 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 14:19:22 +1200
Subject: [PATCH 02/88] s4-dnsserver: Check for too many DNS results
If we had this check in when the wildcard DNS tests were written, we would have
noticed that the name needed to be escaped (see previous commit).
BUG: https://bugzilla.samba.org/show_bug.cgi?id=12994
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Douglas Bagnall <douglas.bagnall at catalyst.net.nz>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/rpc_server/dnsserver/dnsdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/source4/rpc_server/dnsserver/dnsdb.c b/source4/rpc_server/dnsserver/dnsdb.c
index 15fe5657d3a..81a2d2070a0 100644
--- a/source4/rpc_server/dnsserver/dnsdb.c
+++ b/source4/rpc_server/dnsserver/dnsdb.c
@@ -657,6 +657,9 @@ WERROR dnsserver_db_delete_record(TALLOC_CTX *mem_ctx,
if (res->count == 0) {
return WERR_DNS_ERROR_RECORD_DOES_NOT_EXIST;
}
+ if (res->count > 1) {
+ return WERR_DNS_ERROR_RCODE_SERVER_FAILURE;
+ }
el = ldb_msg_find_element(res->msgs[0], "dnsRecord");
if (el == NULL || el->num_values == 0) {
--
2.11.0
From 2deb62f0e51df2520c91314ce1a71e09bebe576a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 28 Aug 2017 14:55:00 +1200
Subject: [PATCH 03/88] ldb_tdb: Make ldb_match_message() available to ldb_tdb
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/common/ldb_match.c | 22 +++++++++++++++-------
lib/ldb/include/ldb_private.h | 19 +++++++++++++++++++
2 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/lib/ldb/common/ldb_match.c b/lib/ldb/common/ldb_match.c
index 1415fac59f9..25fe3f9c21b 100644
--- a/lib/ldb/common/ldb_match.c
+++ b/lib/ldb/common/ldb_match.c
@@ -515,17 +515,18 @@ static int ldb_match_extended(struct ldb_context *ldb,
}
/*
- return 0 if the given parse tree matches the given message. Assumes
- the message is in sorted order
+ Check if a particular message will match the given filter
- return 1 if it matches, and 0 if it doesn't match
+ set *matched to true if it matches, false otherwise
+
+ returns LDB_SUCCESS or an error
this is a recursive function, and does short-circuit evaluation
*/
-static int ldb_match_message(struct ldb_context *ldb,
- const struct ldb_message *msg,
- const struct ldb_parse_tree *tree,
- enum ldb_scope scope, bool *matched)
+int ldb_match_message(struct ldb_context *ldb,
+ const struct ldb_message *msg,
+ const struct ldb_parse_tree *tree,
+ enum ldb_scope scope, bool *matched)
{
unsigned int i;
int ret;
@@ -587,6 +588,13 @@ static int ldb_match_message(struct ldb_context *ldb,
return LDB_ERR_INAPPROPRIATE_MATCHING;
}
+/*
+ return 0 if the given parse tree matches the given message. Assumes
+ the message is in sorted order
+
+ return 1 if it matches, and 0 if it doesn't match
+*/
+
int ldb_match_msg(struct ldb_context *ldb,
const struct ldb_message *msg,
const struct ldb_parse_tree *tree,
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index ae7ec3cfd0d..ab215411a9a 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -287,5 +287,24 @@ int ldb_msg_find_duplicate_val(struct ldb_context *ldb,
const struct ldb_message_element *el,
struct ldb_val **duplicate,
uint32_t options);
+/**
+ Check if a particular message will match the given filter
+
+ \param ldb an ldb context
+ \param msg the message to be checked
+ \param tree the filter tree to check against
+ \param scope the scope to match against
+ (to avoid matching special DNs except on a base search)
+ \param matched a pointer to a boolean set true if it matches,
+ false otherwise
+
+ returns LDB_SUCCESS or an error
+
+ \note this is a recursive function, and does short-circuit evaluation
+ */
+int ldb_match_message(struct ldb_context *ldb,
+ const struct ldb_message *msg,
+ const struct ldb_parse_tree *tree,
+ enum ldb_scope scope, bool *matched);
#endif
--
2.11.0
From a37780359cb12e4c9cfbdbc8b700b595a493a940 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 17:21:28 +1200
Subject: [PATCH 04/88] ldb_tdb: Add helper function
ltdb_search_and_return_base()
This avoids an extra DB lookup for the base, when that is the only
record we will return, and avoids going into the index code for
a base search, as that won't work for special DNs once the GUID
index mode is enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 100 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 95 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index a6c408a53ce..117f6858a17 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -501,6 +501,81 @@ static int ltdb_search_full(struct ltdb_context *ctx)
return ctx->error;
}
+static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
+ struct ltdb_context *ctx)
+{
+ struct ldb_message *msg, *filtered_msg;
+ struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
+ int ret;
+ bool matched;
+
+ msg = ldb_msg_new(ctx);
+ if (!msg) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ret = ltdb_search_dn1(ctx->module, ctx->base, msg,
+ LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
+ LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
+
+ if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ if (ltdb->check_base == false) {
+ /*
+ * In this case, we are done, as no base
+ * checking is allowed in this DB
+ */
+ talloc_free(msg);
+ return LDB_SUCCESS;
+ }
+ ldb_asprintf_errstring(ldb,
+ "No such Base DN: %s",
+ ldb_dn_get_linearized(ctx->base));
+ }
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ret;
+ }
+
+
+ /*
+ * We use this, not ldb_match_msg_error() as we know
+ * we matched on the scope BASE, as we just fetched
+ * the base DN
+ */
+
+ ret = ldb_match_message(ldb, msg,
+ ctx->tree,
+ ctx->scope,
+ &matched);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ret;
+ }
+ if (!matched) {
+ talloc_free(msg);
+ return LDB_SUCCESS;
+ }
+
+ /* filter the attributes that the user wants */
+ ret = ltdb_filter_attrs(ctx, msg, ctx->attrs, &filtered_msg);
+
+ talloc_free(msg);
+
+ if (ret == -1) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ldb_module_send_entry(ctx->req, filtered_msg, NULL);
+ if (ret != LDB_SUCCESS) {
+ /* Regardless of success or failure, the msg
+ * is the callbacks responsiblity, and should
+ * not be talloc_free()'ed */
+ ctx->request_terminated = true;
+ return ret;
+ }
+
+ return LDB_SUCCESS;
+}
+
/*
search the database with a LDAP-like expression.
choses a search method
@@ -532,6 +607,11 @@ int ltdb_search(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
+ ctx->tree = req->op.search.tree;
+ ctx->scope = req->op.search.scope;
+ ctx->base = req->op.search.base;
+ ctx->attrs = req->op.search.attrs;
+
if ((req->op.search.base == NULL) || (ldb_dn_is_null(req->op.search.base) == true)) {
/* Check what we should do with a NULL dn */
@@ -559,6 +639,21 @@ int ltdb_search(struct ltdb_context *ctx)
ldb_dn_get_linearized(req->op.search.base));
ret = LDB_ERR_INVALID_DN_SYNTAX;
+ } else if (req->op.search.scope == LDB_SCOPE_BASE) {
+
+ /*
+ * If we are LDB_SCOPE_BASE, do just one search and
+ * return early. This is critical to ensure we do not
+ * go into the index code for special DNs, as that
+ * will try to look up an index record for a special
+ * record (which doesn't exist).
+ */
+ ret = ltdb_search_and_return_base(ltdb, ctx);
+
+ ltdb_unlock_read(module);
+
+ return ret;
+
} else if (ltdb->check_base) {
/* This database has been marked as 'checkBaseOnSearch', so do a spot check of the base dn */
ret = ltdb_search_base(module, req->op.search.base);
@@ -574,11 +669,6 @@ int ltdb_search(struct ltdb_context *ctx)
ret = LDB_SUCCESS;
}
- ctx->tree = req->op.search.tree;
- ctx->scope = req->op.search.scope;
- ctx->base = req->op.search.base;
- ctx->attrs = req->op.search.attrs;
-
if (ret == LDB_SUCCESS) {
uint32_t match_count = 0;
--
2.11.0
From 838a5f4cefaa80333a73b62b6baa45e4c81a4dfe Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 14:31:18 +1200
Subject: [PATCH 05/88] ldb_tdb: provide ldb_key_dn() and ldb_key_msg()
This will in time allow us to generate a TDB key from
the msg, eg from an objectGUID.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
lib/ldb/ldb_tdb/ldb_search.c | 4 ++--
lib/ldb/ldb_tdb/ldb_tdb.c | 23 ++++++++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.h | 3 ++-
4 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3510dd9b1bc..dec7bcca1cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1625,7 +1625,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
/* check if the DN key has changed, perhaps due to the
case insensitivity of an element changing */
- key2 = ltdb_key(module, msg->dn);
+ key2 = ltdb_key_dn(module, msg->dn);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 117f6858a17..4d3c38a0b08 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -125,7 +125,7 @@ static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
}
/* form the key */
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -214,7 +214,7 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
};
/* form the key */
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OPERATIONS_ERROR;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index ccad8168a6e..0a239d49dd5 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -163,7 +163,7 @@ bool ltdb_key_is_record(TDB_DATA key)
note that the key for a record can depend on whether the
dn refers to a case sensitive index record or not
*/
-TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
TDB_DATA key;
@@ -210,6 +210,19 @@ failed:
}
/*
+ form a TDB_DATA for a record key
+ caller frees
+
+ note that the key for a record can depend on whether the
+ dn refers to a case sensitive index record or not
+*/
+TDB_DATA ltdb_key_msg(struct ldb_module *module,
+ const struct ldb_message *msg)
+{
+ return ltdb_key_dn(module, msg->dn);
+}
+
+/*
check special dn's have valid attributes
currently only @ATTRIBUTES is checked
*/
@@ -297,7 +310,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
- tdb_key = ltdb_key(module, msg->dn);
+ tdb_key = ltdb_key_msg(module, msg);
if (tdb_key.dptr == NULL) {
return LDB_ERR_OTHER;
}
@@ -465,7 +478,7 @@ int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
TDB_DATA tdb_key;
int ret;
- tdb_key = ltdb_key(module, dn);
+ tdb_key = ltdb_key_dn(module, dn);
if (!tdb_key.dptr) {
return LDB_ERR_OTHER;
}
@@ -1093,13 +1106,13 @@ static int ltdb_rename(struct ltdb_context *ctx)
/* We need to, before changing the DB, check if the new DN
* exists, so we can return this error to the caller with an
* unmodified DB */
- tdb_key = ltdb_key(module, req->op.rename.newdn);
+ tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key_old = ltdb_key(module, req->op.rename.olddn);
+ tdb_key_old = ltdb_key_dn(module, req->op.rename.olddn);
if (!tdb_key_old.dptr) {
talloc_free(msg);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index a391606dd16..52e707e9d85 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -115,7 +115,8 @@ int ltdb_unlock_read(struct ldb_module *module);
* index, the old DN index and a possible future ID=
*/
bool ltdb_key_is_record(TDB_DATA key);
-TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
+TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
--
2.11.0
From adf531ae9cbb8873f29dcf6f409f9cc052a82aa6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:01:12 +1200
Subject: [PATCH 06/88] ldb_tdb: Use ltdb_key_msg() in re_index()
This will allow changing to a GUID tdb key in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index dec7bcca1cc..613fc1b4d6d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1623,9 +1623,10 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
return -1;
}
- /* check if the DN key has changed, perhaps due to the
- case insensitivity of an element changing */
- key2 = ltdb_key_dn(module, msg->dn);
+ /* check if the DN key has changed, perhaps due to the case
+ insensitivity of an element changing, or a change from DN
+ to GUID keys */
+ key2 = ltdb_key_msg(module, msg);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
--
2.11.0
From be9a61abb2424eeacfdf6c3b5ed1d445aef838fb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:06:08 +1200
Subject: [PATCH 07/88] ldb_tdb: Add ltdb_search_key()
This allows us to slowly split out the tdb key in the DB from being the DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 41 +++++++++++++++++++++++++++++++----------
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 4d3c38a0b08..bbceead56f0 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -200,25 +200,18 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
+int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
+ struct TDB_DATA tdb_key,
+ struct ldb_message *msg,
unsigned int unpack_flags)
{
- void *data = ldb_module_get_private(module);
- struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
- TDB_DATA tdb_key;
struct ltdb_parse_data_unpack_ctx ctx = {
.msg = msg,
.module = module,
.unpack_flags = unpack_flags
};
- /* form the key */
- tdb_key = ltdb_key_dn(module, dn);
- if (!tdb_key.dptr) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
memset(msg, 0, sizeof(*msg));
msg->num_elements = 0;
@@ -237,6 +230,34 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return ret;
}
+ return LDB_SUCCESS;
+}
+
+/*
+ search the database for a single simple dn, returning all attributes
+ in a single message
+
+ return LDB_ERR_NO_SUCH_OBJECT on record-not-found
+ and LDB_SUCCESS on success
+*/
+int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
+ unsigned int unpack_flags)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ int ret;
+ TDB_DATA tdb_key;
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, dn);
+ if (!tdb_key.dptr) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if ((unpack_flags & LDB_UNPACK_DATA_FLAG_NO_DN) == 0) {
if (!msg->dn) {
msg->dn = ldb_dn_copy(msg, dn);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 52e707e9d85..b4bc330fe92 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -102,6 +102,10 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
+int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
+ struct TDB_DATA tdb_key,
+ struct ldb_message *msg,
+ unsigned int unpack_flags);
int ltdb_filter_attrs(TALLOC_CTX *mem_ctx,
const struct ldb_message *msg, const char * const *attrs,
struct ldb_message **filtered_msg);
--
2.11.0
From 450aee4e9f30a698d442ebf8add5c040e99bd765 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 12:46:57 +1200
Subject: [PATCH 08/88] ldb_tdb: Use a more complete error mapping in
ltdb_search_key()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index bbceead56f0..753eb940048 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -222,10 +222,15 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
talloc_free(tdb_key.dptr);
if (ret == -1) {
- if (tdb_error(ltdb->tdb) == TDB_ERR_NOEXIST) {
- return LDB_ERR_NO_SUCH_OBJECT;
+ ret = ltdb_err_map(tdb_error(ltdb->tdb));
+ if (ret == LDB_SUCCESS) {
+ /*
+ * Just to be sure we don't turn errors
+ * into success
+ */
+ return LDB_ERR_OPERATIONS_ERROR;
}
- return LDB_ERR_OPERATIONS_ERROR;
+ return ret;
} else if (ret != LDB_SUCCESS) {
return ret;
}
--
2.11.0
From 13ea523f7a77265463b4b362c074ef841a849123 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 16:23:33 +1200
Subject: [PATCH 09/88] ldb_tdb: Provide struct ltdb_private to index routines
This will make it easier to switch the GUID index mode on and off
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 90 ++++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.c | 38 +++++++++++++------
lib/ldb/ldb_tdb/ldb_tdb.h | 16 ++++++--
3 files changed, 99 insertions(+), 45 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 613fc1b4d6d..8fd30cfd736 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -79,7 +79,9 @@ static int dn_list_cmp(const struct ldb_val *v1, const struct ldb_val *v2)
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
comparison with the dn returns -1 if not found
*/
-static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_val *v)
+static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
+ const struct dn_list *list,
+ const struct ldb_val *v)
{
unsigned int i;
for (i=0; i<list->count; i++) {
@@ -94,12 +96,14 @@ static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_va
find a entry in a dn_list. Uses a case sensitive comparison with the dn
returns -1 if not found
*/
-static int ltdb_dn_list_find_str(struct dn_list *list, const char *dn)
+static int ltdb_dn_list_find_str(struct ltdb_private *ltdb,
+ struct dn_list *list,
+ const char *dn)
{
struct ldb_val v;
v.data = discard_const_p(unsigned char, dn);
v.length = strlen(dn);
- return ltdb_dn_list_find_val(list, &v);
+ return ltdb_dn_list_find_val(ltdb, list, &v);
}
/*
@@ -219,7 +223,9 @@ normal_index:
/*
save a dn_list into a full @IDX style record
*/
-static int ltdb_dn_list_store_full(struct ldb_module *module, struct ldb_dn *dn,
+static int ltdb_dn_list_store_full(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct dn_list *list)
{
struct ldb_message *msg;
@@ -274,7 +280,8 @@ static int ltdb_dn_list_store(struct ldb_module *module, struct ldb_dn *dn,
struct dn_list *list2;
if (ltdb->idxptr == NULL) {
- return ltdb_dn_list_store_full(module, dn, list);
+ return ltdb_dn_list_store_full(module, ltdb,
+ dn, list);
}
if (ltdb->idxptr->itdb == NULL) {
@@ -345,7 +352,8 @@ static int ltdb_index_traverse_store(struct tdb_context *tdb, TDB_DATA key, TDB_
return -1;
}
- ltdb->idxptr->error = ltdb_dn_list_store_full(module, dn, list);
+ ltdb->idxptr->error = ltdb_dn_list_store_full(module, ltdb,
+ dn, list);
talloc_free(dn);
if (ltdb->idxptr->error != 0) {
return -1;
@@ -575,6 +583,7 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
list = list & list2
*/
static bool list_intersect(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
struct dn_list *list, const struct dn_list *list2)
{
struct dn_list *list3;
@@ -622,7 +631,8 @@ static bool list_intersect(struct ldb_context *ldb,
list3->count = 0;
for (i=0;i<list->count;i++) {
- if (ltdb_dn_list_find_val(list2, &list->dn[i]) != -1) {
+ if (ltdb_dn_list_find_val(ltdb, list2,
+ &list->dn[i]) != -1) {
list3->dn[list3->count] = list->dn[i];
list3->count++;
}
@@ -846,7 +856,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
list->dn = list2->dn;
list->count = list2->count;
found = true;
- } else if (!list_intersect(ldb, list, list2)) {
+ } else if (!list_intersect(ldb, ltdb,
+ list, list2)) {
talloc_free(list2);
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -952,7 +963,8 @@ static int ltdb_index_dn(struct ldb_module *module,
filter a candidate dn_list from an indexed search into a set of results
extracting just the given attributes
*/
-static int ltdb_index_filter(const struct dn_list *dn_list,
+static int ltdb_index_filter(struct ltdb_private *ltdb,
+ const struct dn_list *dn_list,
struct ltdb_context *ac,
uint32_t *match_count)
{
@@ -1123,7 +1135,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
break;
}
- ret = ltdb_index_filter(dn_list, ac, match_count);
+ ret = ltdb_index_filter(ltdb, dn_list, ac, match_count);
talloc_free(dn_list);
return ret;
}
@@ -1148,7 +1160,9 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
*
* @return An ldb error code
*/
-static int ltdb_index_add1(struct ldb_module *module, const char *dn,
+static int ltdb_index_add1(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *el, int v_idx)
{
struct ldb_context *ldb;
@@ -1227,12 +1241,15 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn,
/*
add index entries for one elements in a message
*/
-static int ltdb_index_add_el(struct ldb_module *module, const char *dn,
+static int ltdb_index_add_el(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *el)
{
unsigned int i;
for (i = 0; i < el->num_values; i++) {
- int ret = ltdb_index_add1(module, dn, el, i);
+ int ret = ltdb_index_add1(module, ltdb,
+ dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1244,11 +1261,12 @@ static int ltdb_index_add_el(struct ldb_module *module, const char *dn,
/*
add index entries for all elements in a message
*/
-static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
+static int ltdb_index_add_all(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *dn,
struct ldb_message_element *elements,
unsigned int num_el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
unsigned int i;
if (dn[0] == '@') {
@@ -1265,7 +1283,7 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
- ret = ltdb_index_add_el(module, dn, &elements[i]);
+ ret = ltdb_index_add_el(module, ltdb, dn, &elements[i]);
if (ret != LDB_SUCCESS) {
struct ldb_context *ldb = ldb_module_get_ctx(module);
ldb_asprintf_errstring(ldb,
@@ -1321,9 +1339,9 @@ static int ltdb_index_onelevel(struct ldb_module *module,
el.num_values = 1;
if (add) {
- ret = ltdb_index_add1(module, dn, &el, 0);
+ ret = ltdb_index_add1(module, ltdb, dn, &el, 0);
} else { /* delete */
- ret = ltdb_index_del_value(module, msg->dn, &el, 0);
+ ret = ltdb_index_del_value(module, ltdb, msg->dn, &el, 0);
}
talloc_free(pdn);
@@ -1335,23 +1353,27 @@ static int ltdb_index_onelevel(struct ldb_module *module,
add the index entries for a new element in a record
The caller guarantees that these element values are not yet indexed
*/
-int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_add_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
if (ldb_dn_is_special(dn)) {
return LDB_SUCCESS;
}
if (!ltdb_is_indexed(module, ltdb, el->name)) {
return LDB_SUCCESS;
}
- return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el);
+ return ltdb_index_add_el(module, ltdb,
+ ldb_dn_get_linearized(dn), el);
}
/*
add the index entries for a new record
*/
-int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
+int ltdb_index_add_new(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg)
{
const char *dn;
int ret;
@@ -1365,7 +1387,8 @@ int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, dn, msg->elements,
+ msg->num_elements);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1377,7 +1400,9 @@ int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg)
/*
delete an index entry for one message element
*/
-int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_value(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el, unsigned int v_idx)
{
struct ldb_context *ldb;
@@ -1422,7 +1447,7 @@ int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
return ret;
}
- i = ltdb_dn_list_find_str(list, dn_str);
+ i = ltdb_dn_list_find_str(ltdb, list, dn_str);
if (i == -1) {
/* nothing to delete */
talloc_free(dn_key);
@@ -1452,10 +1477,11 @@ int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
delete the index entries for a element
return -1 on failure
*/
-int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
const char *dn_str;
int ret;
unsigned int i;
@@ -1478,7 +1504,7 @@ int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
return LDB_SUCCESS;
}
for (i = 0; i < el->num_values; i++) {
- ret = ltdb_index_del_value(module, dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1512,7 +1538,8 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
}
for (i = 0; i < msg->num_elements; i++) {
- ret = ltdb_index_del_element(module, msg->dn, &msg->elements[i]);
+ ret = ltdb_index_del_element(module, ltdb,
+ msg->dn, &msg->elements[i]);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1678,6 +1705,8 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
struct ldb_context *ldb;
struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
struct ldb_module *module = ctx->module;
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
struct ldb_message *msg;
const char *dn = NULL;
unsigned int nb_elements_in_db;
@@ -1739,7 +1768,8 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
return -1;
}
- ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, dn,
+ msg->elements, msg->num_elements);
if (ret != LDB_SUCCESS) {
ctx->error = ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 0a239d49dd5..f0855ecba9a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -368,6 +368,7 @@ static bool ldb_tdb_single_valued(const struct ldb_schema_attribute *a,
}
static int ltdb_add_internal(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_message *msg,
bool check_single_value)
{
@@ -432,7 +433,7 @@ static int ltdb_add_internal(struct ldb_module *module,
return ret;
}
- ret = ltdb_index_add_new(module, msg);
+ ret = ltdb_index_add_new(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -449,6 +450,8 @@ static int ltdb_add(struct ltdb_context *ctx)
{
struct ldb_module *module = ctx->module;
struct ldb_request *req = ctx->req;
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret = LDB_SUCCESS;
ret = ltdb_check_special_dn(module, req->op.add.message);
@@ -462,7 +465,8 @@ static int ltdb_add(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_add_internal(module, req->op.add.message, true);
+ ret = ltdb_add_internal(module, ltdb,
+ req->op.add.message, true);
return ret;
}
@@ -622,6 +626,7 @@ static int ltdb_msg_add_element(struct ldb_message *msg,
delete all elements having a specified attribute name
*/
static int msg_delete_attribute(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_message *msg, const char *name)
{
unsigned int i;
@@ -634,7 +639,7 @@ static int msg_delete_attribute(struct ldb_module *module,
}
i = el - msg->elements;
- ret = ltdb_index_del_element(module, msg->dn, el);
+ ret = ltdb_index_del_element(module, ltdb, msg->dn, el);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -656,6 +661,7 @@ static int msg_delete_attribute(struct ldb_module *module,
return LDB Error on failure
*/
static int msg_delete_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_message *msg,
const char *name,
const struct ldb_val *val)
@@ -688,10 +694,11 @@ static int msg_delete_element(struct ldb_module *module,
}
if (matched) {
if (el->num_values == 1) {
- return msg_delete_attribute(module, msg, name);
+ return msg_delete_attribute(module,
+ ltdb, msg, name);
}
- ret = ltdb_index_del_value(module, msg->dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg->dn, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -728,6 +735,8 @@ int ltdb_modify_internal(struct ldb_module *module,
struct ldb_request *req)
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct ldb_message *msg2;
unsigned int i, j;
int ret = LDB_SUCCESS, idx;
@@ -813,7 +822,8 @@ int ltdb_modify_internal(struct ldb_module *module,
ret = LDB_ERR_OTHER;
goto done;
}
- ret = ltdb_index_add_element(module, msg2->dn,
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn,
el);
if (ret != LDB_SUCCESS) {
goto done;
@@ -894,7 +904,8 @@ int ltdb_modify_internal(struct ldb_module *module,
el2->values = vals;
el2->num_values += el->num_values;
- ret = ltdb_index_add_element(module, msg2->dn, el);
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -958,7 +969,8 @@ int ltdb_modify_internal(struct ldb_module *module,
}
/* Delete the attribute if it exists in the DB */
- if (msg_delete_attribute(module, msg2,
+ if (msg_delete_attribute(module, ltdb,
+ msg2,
el->name) != 0) {
ret = LDB_ERR_OTHER;
goto done;
@@ -971,7 +983,8 @@ int ltdb_modify_internal(struct ldb_module *module,
goto done;
}
- ret = ltdb_index_add_element(module, msg2->dn, el);
+ ret = ltdb_index_add_element(module, ltdb,
+ msg2->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -987,7 +1000,9 @@ int ltdb_modify_internal(struct ldb_module *module,
if (msg->elements[i].num_values == 0) {
/* Delete the whole attribute */
- ret = msg_delete_attribute(module, msg2,
+ ret = msg_delete_attribute(module,
+ ltdb,
+ msg2,
msg->elements[i].name);
if (ret == LDB_ERR_NO_SUCH_ATTRIBUTE &&
control_permissive) {
@@ -1004,6 +1019,7 @@ int ltdb_modify_internal(struct ldb_module *module,
/* Delete specified values from an attribute */
for (j=0; j < msg->elements[i].num_values; j++) {
ret = msg_delete_element(module,
+ ltdb,
msg2,
msg->elements[i].name,
&msg->elements[i].values[j]);
@@ -1155,7 +1171,7 @@ static int ltdb_rename(struct ltdb_context *ctx)
* deleted attributes. We could go through all elements but that's
* maybe not the most efficient way
*/
- ret = ltdb_add_internal(module, msg, false);
+ ret = ltdb_add_internal(module, ltdb, msg, false);
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index b4bc330fe92..09c3c5441ab 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -82,13 +82,21 @@ int ltdb_check_at_attributes_values(const struct ldb_val *value);
struct ldb_parse_tree;
int ltdb_search_indexed(struct ltdb_context *ctx, uint32_t *);
-int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg);
+int ltdb_index_add_new(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg);
int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg);
-int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el);
-int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_add_element(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el);
-int ltdb_index_del_value(struct ldb_module *module, struct ldb_dn *dn,
+int ltdb_index_del_value(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *dn,
struct ldb_message_element *el, unsigned int v_idx);
int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
--
2.11.0
From d52cf5f312512ea4aeadd2376d78545328f7e441 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:13:48 +1200
Subject: [PATCH 10/88] ldb_tdb: Add GUID_index_attribute to ltdb_private
This will be used to determine if we are in GUID index mode
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 09c3c5441ab..aa8d162fdec 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -20,6 +20,7 @@ struct ltdb_private {
struct ldb_message *indexlist;
bool one_level_indexes;
bool attribute_indexes;
+ const char *GUID_index_attribute;
} *cache;
int in_transaction;
--
2.11.0
From cac6fb38e677beb4b8af7bd320691cd07e270fe6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:42:25 +1200
Subject: [PATCH 11/88] ldb_tdb: Do not allow a modification of the
GUID_index_attribute (objectGUID)
This would totally break our index scheme if this could be modified.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f0855ecba9a..197b6861f72 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -632,6 +632,17 @@ static int msg_delete_attribute(struct ldb_module *module,
unsigned int i;
int ret;
struct ldb_message_element *el;
+ bool is_special = ldb_dn_is_special(msg->dn);
+
+ if (!is_special
+ && ltdb->cache->GUID_index_attribute != NULL
+ && ldb_attr_cmp(name, ltdb->cache->GUID_index_attribute) == 0) {
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ ldb_asprintf_errstring(ldb, "Must not modify GUID "
+ "attribute %s (used as DB index)",
+ ltdb->cache->GUID_index_attribute);
+ return LDB_ERR_CONSTRAINT_VIOLATION;
+ }
el = ldb_msg_find_element(msg, name);
if (el == NULL) {
--
2.11.0
From 39b7dbb00da2cbfbfb0efd8097519287766ee8e1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:00:48 +1200
Subject: [PATCH 12/88] ldb_tdb: replace strange dn_list_cmp() in index code
This replaces dn_list_cmp() with functions that do not attempt to
to care about string termination. All index values are case sensitive
and correctly length-bound already, even for a DN index
so just use a length check and memcmp()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 27 +++++++++++++++++----------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8fd30cfd736..4db11dfe08e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -61,23 +61,28 @@ int ltdb_index_transaction_start(struct ldb_module *module)
return LDB_SUCCESS;
}
-/* compare two DN entries in a dn_list. Take account of possible
- * differences in string termination */
-static int dn_list_cmp(const struct ldb_val *v1, const struct ldb_val *v2)
+/*
+ see if two ldb_val structures contain exactly the same data
+ return -1 or 1 for a mismatch, 0 for match
+*/
+static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1,
+ const struct ldb_val *v2)
{
- if (v1->length > v2->length && v1->data[v2->length] != 0) {
+ if (v1->length > v2->length) {
return -1;
}
- if (v1->length < v2->length && v2->data[v1->length] != 0) {
+ if (v1->length < v2->length) {
return 1;
}
- return strncmp((char *)v1->data, (char *)v2->data, v1->length);
+ return memcmp(v1->data, v2->data, v1->length);
}
/*
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
- comparison with the dn returns -1 if not found
+ binary-safe comparison for the 'dn' returns -1 if not found
+
+ This is therefore safe when the value is a GUID in the future
*/
static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
const struct dn_list *list,
@@ -85,7 +90,7 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
{
unsigned int i;
for (i=0; i<list->count; i++) {
- if (dn_list_cmp(&list->dn[i], v) == 0) {
+ if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
return i;
}
}
@@ -1053,11 +1058,13 @@ static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
return;
}
- TYPESAFE_QSORT(list->dn, list->count, dn_list_cmp);
+ TYPESAFE_QSORT(list->dn, list->count,
+ ldb_val_equal_exact_for_qsort);
new_count = 1;
for (i=1; i<list->count; i++) {
- if (dn_list_cmp(&list->dn[i], &list->dn[new_count-1]) != 0) {
+ if (ldb_val_equal_exact(&list->dn[i],
+ &list->dn[new_count-1]) == 0) {
if (new_count != i) {
list->dn[new_count] = list->dn[i];
}
--
2.11.0
From 1b06d5a590206de74298e6eb6be0b03b8cf8f32a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:04:43 +1200
Subject: [PATCH 13/88] ldb_tdb: Move constants into ldb_tdb.h
This helps ensure we keep these all in sync.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++--
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 197b6861f72..3c606d61820 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -145,11 +145,12 @@ bool ltdb_key_is_record(TDB_DATA key)
return true;
}
- if (key.dsize < 6) {
+ if (key.dsize < sizeof(LTDB_GUID_KEY_PREFIX)) {
return false;
}
- if (memcmp(key.dptr, "GUID=", 5) == 0) {
+ if (memcmp(key.dptr, LTDB_GUID_KEY_PREFIX,
+ sizeof(LTDB_GUID_KEY_PREFIX) - 1) == 0) {
return true;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index aa8d162fdec..e5e1e192d9f 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -71,6 +71,10 @@ struct ltdb_context {
#define LTDB_MOD_TIMESTAMP "whenChanged"
#define LTDB_OBJECTCLASS "objectClass"
+/* DB keys */
+#define LTDB_GUID_KEY_PREFIX "GUID="
+#define LTDB_GUID_SIZE 16
+
/* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c */
int ltdb_cache_reload(struct ldb_module *module);
--
2.11.0
From 2fc8470e80bd9d87fdce9cbfa8439aff38ad832d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:02:14 +1200
Subject: [PATCH 14/88] ldb_tdb: Store GUID index values in one packed ldb
attribute
This should make them more memory efficient
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 39 +++++++++++++++++++++++++++++++++++++--
1 file changed, 37 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 4db11dfe08e..3ce1d3372cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -264,8 +264,43 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
talloc_free(msg);
return ldb_module_oom(module);
}
- el->values = list->dn;
- el->num_values = list->count;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ el->values = list->dn;
+ el->num_values = list->count;
+ } else {
+ struct ldb_val v;
+ unsigned int i;
+ el->values = talloc_array(msg,
+ struct ldb_val, 1);
+ if (el->values == NULL) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+
+ v.data = talloc_array_size(el->values,
+ list->count,
+ LTDB_GUID_SIZE);
+ if (v.data == NULL) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+
+ v.length = talloc_get_size(v.data);
+
+ for (i = 0; i < list->count; i++) {
+ if (list->dn[i].length !=
+ LTDB_GUID_SIZE) {
+ talloc_free(msg);
+ return ldb_module_operr(module);
+ }
+ memcpy(&v.data[LTDB_GUID_SIZE*i],
+ list->dn[i].data,
+ LTDB_GUID_SIZE);
+ }
+ el->values[0] = v;
+ el->num_values = 1;
+ }
}
ret = ltdb_store(module, msg, TDB_REPLACE);
--
2.11.0
From d0bf82364fd15cfe473665a4d28dae36e7a3bd0f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 17:26:35 +1200
Subject: [PATCH 15/88] ldb_tdb: Read GUID index values as one packed LDB
attribute
This packing should be more efficient to read than the ldb_pack format.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 27 ++++++++++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3ce1d3372cc..f2d619ddfc4 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -215,9 +215,30 @@ normal_index:
* asked for the memory to be allocated on msg, not on each
* value with LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC above
*/
- talloc_steal(el->values, msg);
- list->dn = talloc_steal(list, el->values);
- list->count = el->num_values;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ talloc_steal(el->values, msg);
+ list->dn = talloc_steal(list, el->values);
+ list->count = el->num_values;
+ } else {
+ unsigned int i;
+ static const size_t GUID_val_size = 16;
+ if (el->num_values != 1) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ list->count = el->values[0].length / GUID_val_size;
+ list->dn = talloc_array(list, struct ldb_val, list->count);
+
+ /*
+ * The actual data is on msg, due to
+ * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
+ */
+ talloc_steal(list->dn, msg);
+ for (i = 0; i < list->count; i++) {
+ list->dn[i].data = &el->values[0].data[i * GUID_val_size];
+ list->dn[i].length = GUID_val_size;
+ }
+ }
/* We don't need msg->elements any more */
talloc_free(msg->elements);
--
2.11.0
From 2516883b931156098717d9f2d8b5bf3631d1658c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 15:33:14 +1200
Subject: [PATCH 16/88] ldb_tdb: Refuse to load a GUID index that is not a
multiple of 16 bytes
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f2d619ddfc4..eca5c20498e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -226,6 +226,10 @@ normal_index:
return LDB_ERR_OPERATIONS_ERROR;
}
+ if ((el->values[0].length % GUID_val_size) != 0) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
list->count = el->values[0].length / GUID_val_size;
list->dn = talloc_array(list, struct ldb_val, list->count);
--
2.11.0
From c68c941afe35b882c3aa73db9e2501c554995da0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:04:23 +1200
Subject: [PATCH 17/88] ldb_tdb: Write GUID index values as version 3
Nothing reads these currently, but we should refuse to load a mixed up index
in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index eca5c20498e..1dfb368b364 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -49,6 +49,8 @@ struct ltdb_idxptr {
*/
#define LTDB_INDEXING_VERSION 2
+#define LTDB_GUID_INDEXING_VERSION 3
+
/* enable the idxptr mode when transactions start */
int ltdb_index_transaction_start(struct ldb_module *module)
{
@@ -274,10 +276,20 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
return ldb_module_oom(module);
}
- ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u", LTDB_INDEXING_VERSION);
- if (ret != LDB_SUCCESS) {
- talloc_free(msg);
- return ldb_module_oom(module);
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
+ LTDB_INDEXING_VERSION);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
+ } else {
+ ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
+ LTDB_GUID_INDEXING_VERSION);
+ if (ret != LDB_SUCCESS) {
+ talloc_free(msg);
+ return ldb_module_oom(module);
+ }
}
msg->dn = dn;
--
2.11.0
From e95477838837a149a8445d64cdb3f235fd08cf07 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 09:08:34 +1200
Subject: [PATCH 18/88] ldb_tdb: modify ltdb_delete_noindex() to take a struct
ldb_message
This will make it easier to delete records with the GUID TDB key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 16 +++++++++-------
lib/ldb/ldb_tdb/ldb_tdb.c | 7 ++++---
lib/ldb/ldb_tdb/ldb_tdb.h | 3 ++-
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 1dfb368b364..b5d2ca75031 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -263,19 +263,22 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
struct ldb_message *msg;
int ret;
+ msg = ldb_msg_new(module);
+ if (!msg) {
+ return ldb_module_oom(module);
+ }
+
+ msg->dn = dn;
+
if (list->count == 0) {
- ret = ltdb_delete_noindex(module, dn);
+ ret = ltdb_delete_noindex(module, msg);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ talloc_free(msg);
return LDB_SUCCESS;
}
return ret;
}
- msg = ldb_msg_new(module);
- if (!msg) {
- return ldb_module_oom(module);
- }
-
if (ltdb->cache->GUID_index_attribute == NULL) {
ret = ldb_msg_add_fmt(msg, LTDB_IDXVERSION, "%u",
LTDB_INDEXING_VERSION);
@@ -292,7 +295,6 @@ static int ltdb_dn_list_store_full(struct ldb_module *module,
}
}
- msg->dn = dn;
if (list->count > 0) {
struct ldb_message_element *el;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 3c606d61820..e13b88ff122 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -476,14 +476,15 @@ static int ltdb_add(struct ltdb_context *ctx)
delete a record from the database, not updating indexes (used for deleting
index records)
*/
-int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_delete_noindex(struct ldb_module *module,
+ const struct ldb_message *msg)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA tdb_key;
int ret;
- tdb_key = ltdb_key_dn(module, dn);
+ tdb_key = ltdb_key_dn(module, msg->dn);
if (!tdb_key.dptr) {
return LDB_ERR_OTHER;
}
@@ -516,7 +517,7 @@ static int ltdb_delete_internal(struct ldb_module *module, struct ldb_dn *dn)
goto done;
}
- ret = ltdb_delete_noindex(module, dn);
+ ret = ltdb_delete_noindex(module, msg);
if (ret != LDB_SUCCESS) {
goto done;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index e5e1e192d9f..f28936f247d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -136,7 +136,8 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
-int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
+int ltdb_delete_noindex(struct ldb_module *module,
+ const struct ldb_message *msg);
int ltdb_err_map(enum TDB_ERROR tdb_code);
struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
--
2.11.0
From bf6b210dbe4f910760ad7dc28d7ca3202d977474 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:33:15 +1200
Subject: [PATCH 19/88] ldb_tdb: Delete a successful tdb_store on index add
fail in ltdb_add_internal()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index e13b88ff122..bc81612cb56 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -436,6 +436,17 @@ static int ltdb_add_internal(struct ldb_module *module,
ret = ltdb_index_add_new(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
+ /*
+ * If we failed to index, delete the message again.
+ *
+ * This is particularly important for the GUID index
+ * case, which will only fail for a duplicate DN
+ * in the index add.
+ *
+ * Note that the caller may not cancel the transation
+ * and this means the above add might really show up!
+ */
+ ltdb_delete_noindex(module, msg);
return ret;
}
--
2.11.0
From f276f46d8655f2c22e1f6e503c74a94bfbcbdb97 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 13:08:42 +1200
Subject: [PATCH 20/88] ldb_tdb: Pass the full ldb_message to ldb index
funtions
This allows the objectGUID, rather than the DN, to be the index key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 88 ++++++++++++++++++++++-----------------------
lib/ldb/ldb_tdb/ldb_tdb.c | 10 +++---
lib/ldb/ldb_tdb/ldb_tdb.h | 6 ++--
3 files changed, 52 insertions(+), 52 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b5d2ca75031..c26de922f0a 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -103,13 +103,14 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
find a entry in a dn_list. Uses a case sensitive comparison with the dn
returns -1 if not found
*/
-static int ltdb_dn_list_find_str(struct ltdb_private *ltdb,
+static int ltdb_dn_list_find_msg(struct ltdb_private *ltdb,
struct dn_list *list,
- const char *dn)
+ const struct ldb_message *msg)
{
struct ldb_val v;
- v.data = discard_const_p(unsigned char, dn);
- v.length = strlen(dn);
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ v.data = discard_const_p(unsigned char, dn_str);
+ v.length = strlen(dn_str);
return ltdb_dn_list_find_val(ltdb, list, &v);
}
@@ -1243,7 +1244,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
*/
static int ltdb_index_add1(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, int v_idx)
{
struct ldb_context *ldb;
@@ -1252,6 +1253,7 @@ static int ltdb_index_add1(struct ldb_module *module,
const struct ldb_schema_attribute *a;
struct dn_list *list;
unsigned alloc_len;
+ const char *dn_str;
ldb = ldb_module_get_ctx(module);
@@ -1283,13 +1285,14 @@ static int ltdb_index_add1(struct ldb_module *module,
ldb_debug(ldb, LDB_DEBUG_WARNING,
__location__ ": unique index violation on %s in %s, "
"conficts with %*.*s in %s",
- el->name, dn,
+ el->name, ldb_dn_get_linearized(msg->dn),
(int)list->dn[0].length,
(int)list->dn[0].length,
list->dn[0].data,
ldb_dn_get_linearized(dn_key));
ldb_asprintf_errstring(ldb, __location__ ": unique index violation on %s in %s",
- el->name, dn);
+ el->name,
+ ldb_dn_get_linearized(msg->dn));
talloc_free(list);
return LDB_ERR_ENTRY_ALREADY_EXISTS;
}
@@ -1303,13 +1306,14 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
+ dn_str = ldb_dn_get_linearized(msg->dn);
list->dn[list->count].data
- = (uint8_t *)talloc_strdup(list->dn, dn);
+ = (uint8_t *)talloc_strdup(list->dn, dn_str);
if (list->dn[list->count].data == NULL) {
talloc_free(list);
return LDB_ERR_OPERATIONS_ERROR;
}
- list->dn[list->count].length = strlen(dn);
+ list->dn[list->count].length = strlen(dn_str);
list->count++;
ret = ltdb_dn_list_store(module, dn_key, list);
@@ -1324,13 +1328,13 @@ static int ltdb_index_add1(struct ldb_module *module,
*/
static int ltdb_index_add_el(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
unsigned int i;
for (i = 0; i < el->num_values; i++) {
int ret = ltdb_index_add1(module, ltdb,
- dn, el, i);
+ msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1344,14 +1348,19 @@ static int ltdb_index_add_el(struct ldb_module *module,
*/
static int ltdb_index_add_all(struct ldb_module *module,
struct ltdb_private *ltdb,
- const char *dn,
- struct ldb_message_element *elements,
- unsigned int num_el)
+ const struct ldb_message *msg)
{
+ struct ldb_message_element *elements = msg->elements;
unsigned int i;
+ const char *dn_str;
- if (dn[0] == '@') {
- return LDB_SUCCESS;
+ if (ldb_dn_is_special(msg->dn)) {
+ return LDB_SUCCESS;
+ }
+
+ dn_str = ldb_dn_get_linearized(msg->dn);
+ if (dn_str == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
}
if (!ltdb->cache->attribute_indexes) {
@@ -1359,17 +1368,19 @@ static int ltdb_index_add_all(struct ldb_module *module,
return LDB_SUCCESS;
}
- for (i = 0; i < num_el; i++) {
+ for (i = 0; i < msg->num_elements; i++) {
int ret;
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
- ret = ltdb_index_add_el(module, ltdb, dn, &elements[i]);
+ ret = ltdb_index_add_el(module, ltdb,
+ msg, &elements[i]);
if (ret != LDB_SUCCESS) {
struct ldb_context *ldb = ldb_module_get_ctx(module);
ldb_asprintf_errstring(ldb,
__location__ ": Failed to re-index %s in %s - %s",
- elements[i].name, dn, ldb_errstring(ldb));
+ elements[i].name, dn_str,
+ ldb_errstring(ldb));
return ret;
}
}
@@ -1420,9 +1431,9 @@ static int ltdb_index_onelevel(struct ldb_module *module,
el.num_values = 1;
if (add) {
- ret = ltdb_index_add1(module, ltdb, dn, &el, 0);
+ ret = ltdb_index_add1(module, ltdb, msg, &el, 0);
} else { /* delete */
- ret = ltdb_index_del_value(module, ltdb, msg->dn, &el, 0);
+ ret = ltdb_index_del_value(module, ltdb, msg, &el, 0);
}
talloc_free(pdn);
@@ -1436,17 +1447,16 @@ static int ltdb_index_onelevel(struct ldb_module *module,
*/
int ltdb_index_add_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
- if (ldb_dn_is_special(dn)) {
+ if (ldb_dn_is_special(msg->dn)) {
return LDB_SUCCESS;
}
if (!ltdb_is_indexed(module, ltdb, el->name)) {
return LDB_SUCCESS;
}
- return ltdb_index_add_el(module, ltdb,
- ldb_dn_get_linearized(dn), el);
+ return ltdb_index_add_el(module, ltdb, msg, el);
}
/*
@@ -1456,20 +1466,13 @@ int ltdb_index_add_new(struct ldb_module *module,
struct ltdb_private *ltdb,
const struct ldb_message *msg)
{
- const char *dn;
int ret;
if (ldb_dn_is_special(msg->dn)) {
return LDB_SUCCESS;
}
- dn = ldb_dn_get_linearized(msg->dn);
- if (dn == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- ret = ltdb_index_add_all(module, ltdb, dn, msg->elements,
- msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1483,7 +1486,7 @@ int ltdb_index_add_new(struct ldb_module *module,
*/
int ltdb_index_del_value(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, unsigned int v_idx)
{
struct ldb_context *ldb;
@@ -1492,6 +1495,7 @@ int ltdb_index_del_value(struct ldb_module *module,
int ret, i;
unsigned int j;
struct dn_list *list;
+ struct ldb_dn *dn = msg->dn;
ldb = ldb_module_get_ctx(module);
@@ -1528,7 +1532,7 @@ int ltdb_index_del_value(struct ldb_module *module,
return ret;
}
- i = ltdb_dn_list_find_str(ltdb, list, dn_str);
+ i = ltdb_dn_list_find_msg(ltdb, list, msg);
if (i == -1) {
/* nothing to delete */
talloc_free(dn_key);
@@ -1560,7 +1564,7 @@ int ltdb_index_del_value(struct ldb_module *module,
*/
int ltdb_index_del_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el)
{
const char *dn_str;
@@ -1572,7 +1576,7 @@ int ltdb_index_del_element(struct ldb_module *module,
return LDB_SUCCESS;
}
- dn_str = ldb_dn_get_linearized(dn);
+ dn_str = ldb_dn_get_linearized(msg->dn);
if (dn_str == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1585,7 +1589,7 @@ int ltdb_index_del_element(struct ldb_module *module,
return LDB_SUCCESS;
}
for (i = 0; i < el->num_values; i++) {
- ret = ltdb_index_del_value(module, ltdb, dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1620,7 +1624,7 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
for (i = 0; i < msg->num_elements; i++) {
ret = ltdb_index_del_element(module, ltdb,
- msg->dn, &msg->elements[i]);
+ msg, &msg->elements[i]);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1789,7 +1793,6 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
struct ltdb_private);
struct ldb_message *msg;
- const char *dn = NULL;
unsigned int nb_elements_in_db;
const struct ldb_val val = {
.data = data.dptr,
@@ -1836,8 +1839,6 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
(char *)key.dptr);
talloc_free(msg);
return -1;
- } else {
- dn = ldb_dn_get_linearized(msg->dn);
}
ret = ltdb_index_onelevel(module, msg, 1);
@@ -1849,8 +1850,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
return -1;
}
- ret = ltdb_index_add_all(module, ltdb, dn,
- msg->elements, msg->num_elements);
+ ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
ctx->error = ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index bc81612cb56..1d00574fc8a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -663,7 +663,7 @@ static int msg_delete_attribute(struct ldb_module *module,
}
i = el - msg->elements;
- ret = ltdb_index_del_element(module, ltdb, msg->dn, el);
+ ret = ltdb_index_del_element(module, ltdb, msg, el);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -722,7 +722,7 @@ static int msg_delete_element(struct ldb_module *module,
ltdb, msg, name);
}
- ret = ltdb_index_del_value(module, ltdb, msg->dn, el, i);
+ ret = ltdb_index_del_value(module, ltdb, msg, el, i);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -847,7 +847,7 @@ int ltdb_modify_internal(struct ldb_module *module,
goto done;
}
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn,
+ msg2,
el);
if (ret != LDB_SUCCESS) {
goto done;
@@ -929,7 +929,7 @@ int ltdb_modify_internal(struct ldb_module *module,
el2->num_values += el->num_values;
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn, el);
+ msg2, el);
if (ret != LDB_SUCCESS) {
goto done;
}
@@ -1008,7 +1008,7 @@ int ltdb_modify_internal(struct ldb_module *module,
}
ret = ltdb_index_add_element(module, ltdb,
- msg2->dn, el);
+ msg2, el);
if (ret != LDB_SUCCESS) {
goto done;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f28936f247d..d6b53e8d801 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -93,15 +93,15 @@ int ltdb_index_add_new(struct ldb_module *module,
int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_del_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el);
int ltdb_index_add_element(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el);
int ltdb_index_del_value(struct ldb_module *module,
struct ltdb_private *ltdb,
- struct ldb_dn *dn,
+ const struct ldb_message *msg,
struct ldb_message_element *el, unsigned int v_idx);
int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
--
2.11.0
From 2b8141265a9b75d980599490437c1aaaad5dfc3a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:13:32 +1200
Subject: [PATCH 21/88] ldb_tdb: Optionally use a GUID index key in
ltdb_dn_list_find_msg()
This function is used to find an existing index value and this
change allows it to find the value by GUID rather than by DN once
the GUID index is enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index c26de922f0a..184c4b49e97 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -108,9 +108,19 @@ static int ltdb_dn_list_find_msg(struct ltdb_private *ltdb,
const struct ldb_message *msg)
{
struct ldb_val v;
- const char *dn_str = ldb_dn_get_linearized(msg->dn);
- v.data = discard_const_p(unsigned char, dn_str);
- v.length = strlen(dn_str);
+ const struct ldb_val *key_val;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ v.data = discard_const_p(unsigned char, dn_str);
+ v.length = strlen(dn_str);
+ } else {
+ key_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (key_val == NULL) {
+ return -1;
+ }
+ v = *key_val;
+ }
return ltdb_dn_list_find_val(ltdb, list, &v);
}
--
2.11.0
From e9b37788e7c1f2393040087364d65d1255c9c601 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:18:19 +1200
Subject: [PATCH 22/88] ldb_tdb: Optionally store a GUID as the index record
This allows, when enabled, the index record to contain (say) the objectGUID, not the DN
of the record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 35 +++++++++++++++++++++++++++--------
1 file changed, 27 insertions(+), 8 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 184c4b49e97..d799eaf90f9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1263,7 +1263,6 @@ static int ltdb_index_add1(struct ldb_module *module,
const struct ldb_schema_attribute *a;
struct dn_list *list;
unsigned alloc_len;
- const char *dn_str;
ldb = ldb_module_get_ctx(module);
@@ -1316,14 +1315,34 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn_str = ldb_dn_get_linearized(msg->dn);
- list->dn[list->count].data
- = (uint8_t *)talloc_strdup(list->dn, dn_str);
- if (list->dn[list->count].data == NULL) {
- talloc_free(list);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(msg->dn);
+ list->dn[list->count].data
+ = (uint8_t *)talloc_strdup(list->dn, dn_str);
+ if (list->dn[list->count].data == NULL) {
+ talloc_free(list);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->dn[list->count].length = strlen(dn_str);
+ } else {
+ const struct ldb_val *key_val;
+ key_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (key_val == NULL) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
+
+ if (key_val->length != LTDB_GUID_SIZE) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
+ list->dn[list->count] = ldb_val_dup(list->dn, key_val);
+ if (list->dn[list->count].data == NULL) {
+ talloc_free(list);
+ return ldb_module_operr(module);
+ }
}
- list->dn[list->count].length = strlen(dn_str);
list->count++;
ret = ltdb_dn_list_store(module, dn_key, list);
--
2.11.0
From f857e46af44c5b0bd000f6f72dc55b5cc2cb61a8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 14:39:08 +1200
Subject: [PATCH 23/88] ldb_tdb: Implement ltdb_search_base() for a GUID index
The GUID index case can not directly use ltdb_key_dn() and tdb_exists() to
show that a records exists.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 32 +++++++++++++++++++++++---------
1 file changed, 23 insertions(+), 9 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 753eb940048..661d9704422 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -115,24 +115,38 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
*/
static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
{
- void *data = ldb_module_get_private(module);
- struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- TDB_DATA tdb_key;
int exists;
+ int ret;
+ struct ldb_message *msg = NULL;
if (ldb_dn_is_null(dn)) {
return LDB_ERR_NO_SUCH_OBJECT;
}
- /* form the key */
- tdb_key = ltdb_key_dn(module, dn);
- if (!tdb_key.dptr) {
+ /*
+ * We can't use tdb_exists() directly on a key when the TDB
+ * key is the GUID one, not the DN based one. So we just do a
+ * normal search and avoid most of the allocation with the
+ * LDB_UNPACK_DATA_FLAG_NO_DN and
+ * LDB_UNPACK_DATA_FLAG_NO_ATTRS flags
+ */
+ msg = ldb_msg_new(module);
+ if (msg == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
- exists = tdb_exists(ltdb->tdb, tdb_key);
- talloc_free(tdb_key.dptr);
-
+ ret = ltdb_search_dn1(module, dn,
+ msg,
+ LDB_UNPACK_DATA_FLAG_NO_DN|
+ LDB_UNPACK_DATA_FLAG_NO_ATTRS);
+ talloc_free(msg);
+ if (ret == LDB_SUCCESS) {
+ exists = true;
+ } else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ exists = false;
+ } else {
+ return ret;
+ }
if (exists) {
return LDB_SUCCESS;
}
--
2.11.0
From 900ea15b82f1eb772d985233a3a05132ea578635 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 11:33:17 +1200
Subject: [PATCH 24/88] ldb_tdb: Prepare to handle rename with GUID index by
using ltdb_search_base()
This will allow use of a GUID TDB key in the future. While ltdb_search_base()
might be marginally slower than tdb_exists(), no allocation is done for the
attributes or DN, and renmaes are not a very common operation.
This allows a check if the target DN exists even when the direct DN -> key
link is broken.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 2 +-
lib/ldb/ldb_tdb/ldb_tdb.c | 44 ++++++++++++++++++++++++++++++++------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
3 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 661d9704422..555e7cb0b66 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -113,7 +113,7 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-static int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
{
int exists;
int ret;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1d00574fc8a..7dd14202d38 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1145,7 +1145,11 @@ static int ltdb_rename(struct ltdb_context *ctx)
/* We need to, before changing the DB, check if the new DN
* exists, so we can return this error to the caller with an
- * unmodified DB */
+ * unmodified DB
+ *
+ * Even in GUID index mode we use ltdb_key_dn() as we are
+ * trying to figure out if this is just a case rename
+ */
tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
@@ -1159,19 +1163,35 @@ static int ltdb_rename(struct ltdb_context *ctx)
return LDB_ERR_OPERATIONS_ERROR;
}
- /* Only declare a conflict if the new DN already exists, and it isn't a case change on the old DN */
- if (tdb_key_old.dsize != tdb_key.dsize || memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
- if (tdb_exists(ltdb->tdb, tdb_key)) {
- talloc_free(tdb_key_old.dptr);
- talloc_free(tdb_key.dptr);
- ldb_asprintf_errstring(ldb_module_get_ctx(module),
- "Entry %s already exists",
- ldb_dn_get_linearized(req->op.rename.newdn));
- /* finding the new record already in the DB is an error */
- talloc_free(msg);
- return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ /*
+ * Only declare a conflict if the new DN already exists,
+ * and it isn't a case change on the old DN
+ */
+ if (tdb_key_old.dsize != tdb_key.dsize
+ || memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
+ ret = ltdb_search_base(module,
+ req->op.rename.newdn);
+ if (ret == LDB_SUCCESS) {
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ } else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+ ret = LDB_SUCCESS;
}
}
+
+ /* finding the new record already in the DB is an error */
+
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Entry %s already exists",
+ ldb_dn_get_linearized(req->op.rename.newdn));
+ }
+ if (ret != LDB_SUCCESS) {
+ talloc_free(tdb_key_old.dptr);
+ talloc_free(tdb_key.dptr);
+ talloc_free(msg);
+ return ret;
+ }
+
talloc_free(tdb_key_old.dptr);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index d6b53e8d801..e9d74664922 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -115,6 +115,7 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
+int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn);
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
struct TDB_DATA tdb_key,
struct ldb_message *msg,
--
2.11.0
From 489155ec0b568d5435c4fa3a5895fb389df2662b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 12:58:58 +1200
Subject: [PATCH 25/88] ldb_tdb: Split ltdb_index_onelevel() into a helper
function
This will allow the code to be re-used for storing the DN->GUID index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 74 ++++++++++++++++++++++++++++-----------------
1 file changed, 47 insertions(+), 27 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index d799eaf90f9..f41ce258ba4 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1419,43 +1419,25 @@ static int ltdb_index_add_all(struct ldb_module *module,
/*
- insert a one level index for a message
+ insert a DN index for a message
*/
-static int ltdb_index_onelevel(struct ldb_module *module,
- const struct ldb_message *msg, int add)
-{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
- struct ltdb_private);
+static int ltdb_modify_index_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_message *msg,
+ struct ldb_dn *dn,
+ const char *index, int add)
+{
struct ldb_message_element el;
struct ldb_val val;
- struct ldb_dn *pdn;
- const char *dn;
int ret;
- /* We index for ONE Level only if requested */
- if (!ltdb->cache->one_level_indexes) {
- return LDB_SUCCESS;
- }
-
- pdn = ldb_dn_get_parent(module, msg->dn);
- if (pdn == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- dn = ldb_dn_get_linearized(msg->dn);
- if (dn == NULL) {
- talloc_free(pdn);
- return LDB_ERR_OPERATIONS_ERROR;
- }
-
- val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(pdn));
+ val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
if (val.data == NULL) {
- talloc_free(pdn);
return LDB_ERR_OPERATIONS_ERROR;
}
val.length = strlen((char *)val.data);
- el.name = LTDB_IDXONE;
+ el.name = index;
el.values = &val;
el.num_values = 1;
@@ -1465,6 +1447,44 @@ static int ltdb_index_onelevel(struct ldb_module *module,
ret = ltdb_index_del_value(module, ltdb, msg, &el, 0);
}
+ if (ret != LDB_SUCCESS) {
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb,
+ __location__
+ ": Failed to modify %s "
+ "against %s in %s - %s",
+ index,
+ ltdb->cache->GUID_index_attribute,
+ dn_str, ldb_errstring(ldb));
+ return ret;
+ }
+ return ret;
+}
+
+/*
+ insert a one level index for a message
+*/
+static int ltdb_index_onelevel(struct ldb_module *module,
+ const struct ldb_message *msg, int add)
+{
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
+ struct ldb_dn *pdn;
+ int ret;
+
+ /* We index for ONE Level only if requested */
+ if (!ltdb->cache->one_level_indexes) {
+ return LDB_SUCCESS;
+ }
+
+ pdn = ldb_dn_get_parent(module, msg->dn);
+ if (pdn == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ret = ltdb_modify_index_dn(module, ltdb,
+ msg, pdn, LTDB_IDXONE, add);
+
talloc_free(pdn);
return ret;
--
2.11.0
From c616ae57390e06b9b1175ebf1163559822b555f6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 12:25:42 +1200
Subject: [PATCH 26/88] ldb_tdb: Add/remove a GUID index of the DN during
ltdb_index_add_all()/ltdb_index_delete()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 41 +++++++++++++++++++++++++++++++++++++----
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
2 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f41ce258ba4..2a95af95086 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -44,6 +44,10 @@ struct ltdb_idxptr {
int error;
};
+static int ltdb_write_index_dn_guid(struct ldb_module *module,
+ const struct ldb_message *msg,
+ int add);
+
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
*/
@@ -1382,23 +1386,28 @@ static int ltdb_index_add_all(struct ldb_module *module,
struct ldb_message_element *elements = msg->elements;
unsigned int i;
const char *dn_str;
+ int ret;
- if (ldb_dn_is_special(msg->dn)) {
- return LDB_SUCCESS;
- }
+ if (ldb_dn_is_special(msg->dn)) {
+ return LDB_SUCCESS;
+ }
dn_str = ldb_dn_get_linearized(msg->dn);
if (dn_str == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
+ ret = ltdb_write_index_dn_guid(module, msg, 1);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if (!ltdb->cache->attribute_indexes) {
/* no indexed fields */
return LDB_SUCCESS;
}
for (i = 0; i < msg->num_elements; i++) {
- int ret;
if (!ltdb_is_indexed(module, ltdb, elements[i].name)) {
continue;
}
@@ -1491,6 +1500,25 @@ static int ltdb_index_onelevel(struct ldb_module *module,
}
/*
+ insert a one level index for a message
+*/
+static int ltdb_write_index_dn_guid(struct ldb_module *module,
+ const struct ldb_message *msg,
+ int add)
+{
+ struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
+ struct ltdb_private);
+
+ /* We index for DN only if using a GUID index */
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ return LDB_SUCCESS;
+ }
+
+ return ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
+ LTDB_IDXDN, add);
+}
+
+/*
add the index entries for a new element in a record
The caller guarantees that these element values are not yet indexed
*/
@@ -1666,6 +1694,11 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
return ret;
}
+ ret = ltdb_write_index_dn_guid(module, msg, 0);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
if (!ltdb->cache->attribute_indexes) {
/* no indexed fields */
return LDB_SUCCESS;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index e9d74664922..75759c697a2 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -60,6 +60,7 @@ struct ltdb_context {
#define LTDB_IDXVERSION "@IDXVERSION"
#define LTDB_IDXATTR "@IDXATTR"
#define LTDB_IDXONE "@IDXONE"
+#define LTDB_IDXDN "@IDXDN"
#define LTDB_BASEINFO "@BASEINFO"
#define LTDB_OPTIONS "@OPTIONS"
#define LTDB_ATTRIBUTES "@ATTRIBUTES"
--
2.11.0
From f6ec1240f25ff13f709a4b5146954a6ad2667bf8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:21:00 +1200
Subject: [PATCH 27/88] ldb_tdb: Give LDB_ERR_ENTRY_ALREADY_EXISTS when a
duplicate DN index is detected
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 2a95af95086..e7f71dc79d5 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1506,6 +1506,7 @@ static int ltdb_write_index_dn_guid(struct ldb_module *module,
const struct ldb_message *msg,
int add)
{
+ int ret;
struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
struct ltdb_private);
@@ -1514,8 +1515,16 @@ static int ltdb_write_index_dn_guid(struct ldb_module *module,
return LDB_SUCCESS;
}
- return ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
- LTDB_IDXDN, add);
+ ret = ltdb_modify_index_dn(module, ltdb, msg, msg->dn,
+ LTDB_IDXDN, add);
+
+ if (ret == LDB_ERR_CONSTRAINT_VIOLATION) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Entry %s already exists",
+ ldb_dn_get_linearized(msg->dn));
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ }
+ return ret;
}
/*
--
2.11.0
From e738c76cc168798a0eb9eae5f4579be263e9478a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:21:26 +1200
Subject: [PATCH 28/88] ldb_tdb: Give LDB_ERR_CONSTRAINT_VIOLATION when a
duplicate GUID index is detected
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 7dd14202d38..c86faa5d6fd 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -328,7 +328,18 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
ret = tdb_store(ltdb->tdb, tdb_key, tdb_data, flgs);
if (ret != 0) {
+ bool is_special = ldb_dn_is_special(msg->dn);
ret = ltdb_err_map(tdb_error(ltdb->tdb));
+
+ /*
+ * LDB_ERR_ENTRY_ALREADY_EXISTS means the DN, not
+ * the GUID, so re-map
+ */
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS
+ && !is_special
+ && ltdb->cache->GUID_index_attribute != NULL) {
+ ret = LDB_ERR_CONSTRAINT_VIOLATION;
+ }
goto done;
}
--
2.11.0
From c9611161e04dbec47338449458d33991889691f5 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 13:26:30 +1200
Subject: [PATCH 29/88] ldb_tdb: Pass ltdb_private to ltdb_dn_list_load()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index e7f71dc79d5..67ff3fe0e32 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -166,12 +166,12 @@ static struct dn_list *ltdb_index_idxptr(struct ldb_module *module, TDB_DATA rec
struct dn_list
*/
static int ltdb_dn_list_load(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_dn *dn, struct dn_list *list)
{
struct ldb_message *msg;
int ret;
struct ldb_message_element *el;
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private);
TDB_DATA rec;
struct dn_list *list2;
TDB_DATA key;
@@ -635,7 +635,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
dn = ltdb_index_key(ldb, tree->u.equality.attr, &tree->u.equality.value, NULL);
if (!dn) return LDB_ERR_OPERATIONS_ERROR;
- ret = ltdb_dn_list_load(module, dn, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn, list);
talloc_free(dn);
return ret;
}
@@ -979,6 +979,7 @@ static int ltdb_index_dn_and(struct ldb_module *module,
return a list of matching objects using a one-level index
*/
static int ltdb_index_dn_one(struct ldb_module *module,
+ struct ltdb_private *ltdb,
struct ldb_dn *parent_dn,
struct dn_list *list)
{
@@ -998,7 +999,7 @@ static int ltdb_index_dn_one(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_dn_list_load(module, key, list);
+ ret = ltdb_dn_list_load(module, ltdb, key, list);
talloc_free(key);
if (ret != LDB_SUCCESS) {
return ret;
@@ -1209,7 +1210,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_dn_one(ac->module, ac->base, dn_list);
+ ret = ltdb_index_dn_one(ac->module, ltdb, ac->base, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
return ret;
@@ -1282,7 +1283,7 @@ static int ltdb_index_add1(struct ldb_module *module,
}
talloc_steal(list, dn_key);
- ret = ltdb_dn_list_load(module, dn_key, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
if (ret != LDB_SUCCESS && ret != LDB_ERR_NO_SUCH_OBJECT) {
talloc_free(list);
return ret;
@@ -1605,7 +1606,7 @@ int ltdb_index_del_value(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_dn_list_load(module, dn_key, list);
+ ret = ltdb_dn_list_load(module, ltdb, dn_key, list);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* it wasn't indexed. Did we have an earlier error? If we did then
its gone now */
--
2.11.0
From de57e7c408330e264fc3cb5a9fc4a300d3a9d18c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 17:51:40 +1200
Subject: [PATCH 30/88] ldb_tdb: Do not directly assign DN into the index
result list
With the GUID index option, the values in the index result list may not be a DN
but the objectGUID. We look up the @IDXDN index to get that if required.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 91 +++++++++++++++++++++++++++++++++------------
1 file changed, 68 insertions(+), 23 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 67ff3fe0e32..b0a8f723fc0 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -651,6 +651,8 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
const struct ldb_parse_tree *tree,
struct dn_list *list)
{
+ /* Must be at the same scope as the ltdb_index_dn_simple */
+ struct ldb_parse_tree dn_tree;
if (ltdb->disallow_dn_filter &&
(ldb_attr_cmp(tree->u.equality.attr, "dn") == 0)) {
/* in AD mode we do not support "(dn=...)" search filters */
@@ -659,14 +661,21 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
return LDB_SUCCESS;
}
if (ldb_attr_dn(tree->u.equality.attr) == 0) {
- list->dn = talloc_array(list, struct ldb_val, 1);
- if (list->dn == NULL) {
- ldb_module_oom(module);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ list->dn = talloc_array(list, struct ldb_val, 1);
+ if (list->dn == NULL) {
+ ldb_module_oom(module);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->dn[0] = tree->u.equality.value;
+ list->count = 1;
+ return LDB_SUCCESS;
}
- list->dn[0] = tree->u.equality.value;
- list->count = 1;
- return LDB_SUCCESS;
+
+ /* Overwrite the attr with @IDXDN */
+ dn_tree = *tree;
+ dn_tree.u.equality.attr = LTDB_IDXDN;
+ tree = &dn_tree;
}
return ltdb_index_dn_simple(module, ltdb, tree, list);
}
@@ -978,10 +987,11 @@ static int ltdb_index_dn_and(struct ldb_module *module,
/*
return a list of matching objects using a one-level index
*/
-static int ltdb_index_dn_one(struct ldb_module *module,
- struct ltdb_private *ltdb,
- struct ldb_dn *parent_dn,
- struct dn_list *list)
+static int ltdb_index_dn_attr(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *attr,
+ struct ldb_dn *dn,
+ struct dn_list *list)
{
struct ldb_context *ldb;
struct ldb_dn *key;
@@ -991,9 +1001,9 @@ static int ltdb_index_dn_one(struct ldb_module *module,
ldb = ldb_module_get_ctx(module);
/* work out the index key from the parent DN */
- val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(parent_dn));
+ val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
val.length = strlen((char *)val.data);
- key = ltdb_index_key(ldb, LTDB_IDXONE, &val, NULL);
+ key = ltdb_index_key(ldb, attr, &val, NULL);
if (!key) {
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1013,6 +1023,47 @@ static int ltdb_index_dn_one(struct ldb_module *module,
}
/*
+ return a list of matching objects using a one-level index
+ */
+static int ltdb_index_dn_one(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *parent_dn,
+ struct dn_list *list)
+{
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXONE, parent_dn, list);
+}
+
+/*
+ return a list of matching objects using the DN index
+ */
+static int ltdb_index_dn_base_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *base_dn,
+ struct dn_list *dn_list)
+{
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXDN, base_dn, dn_list);
+ }
+
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ talloc_free(dn_list);
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = discard_const_p(unsigned char,
+ ldb_dn_get_linearized(base_dn));
+ if (dn_list->dn[0].data == NULL) {
+ talloc_free(dn_list);
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
+ dn_list->count = 1;
+ return LDB_SUCCESS;
+}
+
+/*
return a list of dn's that might match a indexed search or
an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
*/
@@ -1191,18 +1242,12 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
switch (ac->scope) {
case LDB_SCOPE_BASE:
- dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
- if (dn_list->dn == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(ac->module);
- }
- dn_list->dn[0].data = discard_const_p(unsigned char, ldb_dn_get_linearized(ac->base));
- if (dn_list->dn[0].data == NULL) {
+ ret = ltdb_index_dn_base_dn(ac->module, ltdb,
+ ac->base, dn_list);
+ if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
- return ldb_module_oom(ac->module);
+ return ret;
}
- dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
- dn_list->count = 1;
break;
case LDB_SCOPE_ONELEVEL:
--
2.11.0
From 432a02e6f5d7087893fe3faccd060b9a14180d60 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:33:24 +1200
Subject: [PATCH 31/88] ldb_tdb: add control points for the new GUID index mode
The @IDXGUID attribute in the @INDEXLIST will be objectGUID
in Samba.
The @IDX_DN_GUID attribute in the @INDEXLIST will be GUID
in Samba.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 75759c697a2..4ef4271df9a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -61,6 +61,8 @@ struct ltdb_context {
#define LTDB_IDXATTR "@IDXATTR"
#define LTDB_IDXONE "@IDXONE"
#define LTDB_IDXDN "@IDXDN"
+#define LTDB_IDXGUID "@IDXGUID"
+#define LTDB_IDX_DN_GUID "@IDX_DN_GUID"
#define LTDB_BASEINFO "@BASEINFO"
#define LTDB_OPTIONS "@OPTIONS"
#define LTDB_ATTRIBUTES "@ATTRIBUTES"
--
2.11.0
From 43bd0ee2de9ebf2f67e62e71c9ecdf2c00381e51 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 10:42:40 +1200
Subject: [PATCH 32/88] ldb_tdb: Load the syntax of the GUID index attr during
ltdb_cache_load()
This allows us to use the ldif_write function later to create a string GUID for the TDB key.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_cache.c | 12 ++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 2 ++
2 files changed, 14 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index f08e073af1a..137ddf97fe8 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -371,6 +371,7 @@ int ltdb_cache_load(struct ldb_module *module)
struct ldb_dn *baseinfo_dn = NULL, *options_dn = NULL;
uint64_t seq;
struct ldb_message *baseinfo = NULL, *options = NULL;
+ const struct ldb_schema_attribute *a;
int r;
ldb = ldb_module_get_ctx(module);
@@ -474,6 +475,17 @@ int ltdb_cache_load(struct ldb_module *module)
goto failed;
}
+ ltdb->GUID_index_syntax = NULL;
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ /*
+ * Now the attributes are loaded, set the guid_index_syntax.
+ * This can't fail, it will return a default at worst
+ */
+ a = ldb_schema_attribute_by_name(ldb,
+ ltdb->cache->GUID_index_attribute);
+ ltdb->GUID_index_syntax = a->syntax;
+ }
+
done:
talloc_free(options);
talloc_free(baseinfo);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 4ef4271df9a..db73632bf93 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,8 @@ struct ltdb_private {
bool warn_unindexed;
bool warn_reindex;
+
+ const struct ldb_schema_syntax *GUID_index_syntax;
};
struct ltdb_context {
--
2.11.0
From c1be83ad753e9e0f3fdeec537355a1d84df11ae9 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 16 Aug 2017 10:44:34 +1200
Subject: [PATCH 33/88] ldb_tdb: Add a function to take a GUID and make the
TDB_DATA key
This allows us to format the TDB key as DN=GUID=f7c953ee-cf9c-433f-b423-21ce04d09591
and so be compatible with an un-indexed search and a re-index with an old ldb.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 26 ++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
2 files changed, 30 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index c86faa5d6fd..6392201694e 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -210,6 +210,32 @@ failed:
return key;
}
+TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *GUID_val)
+{
+ TDB_DATA key;
+ const char *GUID_prefix = "GUID=";
+ const int GUID_prefix_len = strlen(GUID_prefix);
+
+ key.dptr = talloc_size(mem_ctx,
+ GUID_val->length+GUID_prefix_len);
+
+ if (key.dptr == NULL) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ memcpy(key.dptr, "GUID=", GUID_prefix_len);
+ memcpy(&key.dptr[GUID_prefix_len],
+ GUID_val->data, GUID_val->length);
+
+ key.dsize = talloc_get_size(key.dptr);
+ return key;
+}
+
/*
form a TDB_DATA for a record key
caller frees
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index db73632bf93..5aa0d5f35fb 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -140,6 +140,10 @@ int ltdb_unlock_read(struct ldb_module *module);
bool ltdb_key_is_record(TDB_DATA key);
TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
+TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *guid_val);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From 7fbcfc0cf36a66439f6265138fd5256f14deb310 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 11 Aug 2017 18:09:01 +1200
Subject: [PATCH 34/88] ldb_tdb: Add a function to get the GUID key for a DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 40 ++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 5 +++++
2 files changed, 45 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b0a8f723fc0..272b622db0c 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -47,6 +47,10 @@ struct ltdb_idxptr {
static int ltdb_write_index_dn_guid(struct ldb_module *module,
const struct ldb_message *msg,
int add);
+static int ltdb_index_dn_base_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ struct ldb_dn *base_dn,
+ struct dn_list *dn_list);
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
@@ -266,6 +270,42 @@ normal_index:
return LDB_SUCCESS;
}
+int ltdb_key_dn_from_idx(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ TDB_DATA *tdb_key)
+{
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ int ret;
+ struct dn_list *list = talloc(mem_ctx, struct dn_list);
+ if (list == NULL) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ret = ltdb_index_dn_base_dn(module, ltdb, dn, list);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
+ if (list->count == 0) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ if (list->count > 1) {
+ return LDB_ERR_CONSTRAINT_VIOLATION;
+ }
+
+ *tdb_key = ltdb_guid_to_key(module, ltdb,
+ mem_ctx, &list->dn[0]);
+ if (tdb_key->dptr == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ return LDB_SUCCESS;
+}
+
+
/*
save a dn_list into a full @IDX style record
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 5aa0d5f35fb..853ae7d3b9c 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -112,6 +112,11 @@ int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
int ltdb_index_transaction_commit(struct ldb_module *module);
int ltdb_index_transaction_cancel(struct ldb_module *module);
+int ltdb_key_dn_from_idx(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ TDB_DATA *tdb_key);
/* The following definitions come from lib/ldb/ldb_tdb/ldb_search.c */
--
2.11.0
From 7be1d6d86815d8f1bbddb488763c1bb0efe70b48 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 16:13:42 +1200
Subject: [PATCH 35/88] ldb_tdb: Add an index shortcut for a <GUID= DN
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 47 ++++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 272b622db0c..07a018beb0d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1082,25 +1082,42 @@ static int ltdb_index_dn_base_dn(struct ldb_module *module,
struct ldb_dn *base_dn,
struct dn_list *dn_list)
{
- if (ltdb->cache->GUID_index_attribute != NULL) {
- return ltdb_index_dn_attr(module, ltdb,
- LTDB_IDXDN, base_dn, dn_list);
+ const struct ldb_val *guid_val = NULL;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = discard_const_p(unsigned char,
+ ldb_dn_get_linearized(base_dn));
+ if (dn_list->dn[0].data == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
+ dn_list->count = 1;
+
+ return LDB_SUCCESS;
}
- dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
- if (dn_list->dn == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(module);
+ if (ltdb->cache->GUID_index_dn_component != NULL) {
+ guid_val = ldb_dn_get_extended_component(base_dn,
+ ltdb->cache->GUID_index_dn_component);
}
- dn_list->dn[0].data = discard_const_p(unsigned char,
- ldb_dn_get_linearized(base_dn));
- if (dn_list->dn[0].data == NULL) {
- talloc_free(dn_list);
- return ldb_module_oom(module);
+
+ if (guid_val != NULL) {
+ dn_list->dn = talloc_array(dn_list, struct ldb_val, 1);
+ if (dn_list->dn == NULL) {
+ return ldb_module_oom(module);
+ }
+ dn_list->dn[0].data = guid_val->data;
+ dn_list->dn[0].length = guid_val->length;
+ dn_list->count = 1;
+
+ return LDB_SUCCESS;
}
- dn_list->dn[0].length = strlen((char *)dn_list->dn[0].data);
- dn_list->count = 1;
- return LDB_SUCCESS;
+
+ return ltdb_index_dn_attr(module, ltdb,
+ LTDB_IDXDN, base_dn, dn_list);
}
/*
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 853ae7d3b9c..1e639e9a60a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -21,6 +21,7 @@ struct ltdb_private {
bool one_level_indexes;
bool attribute_indexes;
const char *GUID_index_attribute;
+ const char *GUID_index_dn_component;
} *cache;
int in_transaction;
--
2.11.0
From 22f625da6b1c109f425589b78b29f0b649fc0666 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 17:27:23 +1200
Subject: [PATCH 36/88] ldb_tdb: Check version number on index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 29 ++++++++++++++++++++++++++---
1 file changed, 26 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 07a018beb0d..1d49533be79 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -174,7 +174,7 @@ static int ltdb_dn_list_load(struct ldb_module *module,
struct ldb_dn *dn, struct dn_list *list)
{
struct ldb_message *msg;
- int ret;
+ int ret, version;
struct ldb_message_element *el;
TDB_DATA rec;
struct dn_list *list2;
@@ -222,14 +222,14 @@ normal_index:
return ret;
}
- /* TODO: check indexing version number */
-
el = ldb_msg_find_element(msg, LTDB_IDX);
if (!el) {
talloc_free(msg);
return LDB_SUCCESS;
}
+ version = ldb_msg_find_attr_as_int(msg, LTDB_IDXVERSION, 0);
+
/*
* we avoid copying the strings by stealing the list. We have
* to steal msg onto el->values (which looks odd) because we
@@ -237,12 +237,35 @@ normal_index:
* value with LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC above
*/
if (ltdb->cache->GUID_index_attribute == NULL) {
+ /* check indexing version number */
+ if (version != LTDB_INDEXING_VERSION) {
+ ldb_debug_set(ldb_module_get_ctx(module),
+ LDB_DEBUG_ERROR,
+ "Wrong DN index version %d "
+ "expected %d for %s",
+ version, LTDB_INDEXING_VERSION,
+ ldb_dn_get_linearized(dn));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
talloc_steal(el->values, msg);
list->dn = talloc_steal(list, el->values);
list->count = el->num_values;
} else {
unsigned int i;
static const size_t GUID_val_size = 16;
+ if (version != LTDB_GUID_INDEXING_VERSION) {
+ /* This is quite likely during the DB startup
+ on first upgrade to using a GUID index */
+ ldb_debug_set(ldb_module_get_ctx(module),
+ LDB_DEBUG_ERROR,
+ "Wrong GUID index version %d "
+ "expected %d for %s",
+ version, LTDB_GUID_INDEXING_VERSION,
+ ldb_dn_get_linearized(dn));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
if (el->num_values != 1) {
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From c371520e3f7e3fa8608320b2df96357b0b9b5482 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 15:47:15 +1200
Subject: [PATCH 37/88] ldb_tdb: Add mem_ctx to ltdb_key_dn() and
ltdb_key_msg()
This follows modern Samba coding style where memory
returned is allocated on a supplied memory context.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
lib/ldb/ldb_tdb/ldb_search.c | 15 ++++++++++++---
lib/ldb/ldb_tdb/ldb_tdb.c | 43 ++++++++++++++++++++++++++++---------------
lib/ldb/ldb_tdb/ldb_tdb.h | 6 ++++--
4 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 1d49533be79..f6dfb8fc3c9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1955,7 +1955,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
/* check if the DN key has changed, perhaps due to the case
insensitivity of an element changing, or a change from DN
to GUID keys */
- key2 = ltdb_key_msg(module, msg);
+ key2 = ltdb_key_msg(module, msg, msg);
if (key2.dptr == NULL) {
/* probably a corrupt record ... darn */
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid DN in re_index: %s",
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 555e7cb0b66..350f01b8093 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -215,7 +215,7 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
and LDB_SUCCESS on success
*/
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
- struct TDB_DATA tdb_key,
+ const struct TDB_DATA tdb_key,
struct ldb_message *msg,
unsigned int unpack_flags)
{
@@ -233,7 +233,6 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
ret = tdb_parse_record(ltdb->tdb, tdb_key,
ltdb_parse_data_unpack, &ctx);
- talloc_free(tdb_key.dptr);
if (ret == -1) {
ret = ltdb_err_map(tdb_error(ltdb->tdb));
@@ -266,13 +265,23 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
TDB_DATA tdb_key;
+
+ TALLOC_CTX *tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
- tdb_key = ltdb_key_dn(module, dn);
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OPERATIONS_ERROR;
}
ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
+
+ TALLOC_FREE(tdb_key_ctx);
+
if (ret != LDB_SUCCESS) {
return ret;
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 6392201694e..f81fc054c3a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -164,9 +164,9 @@ bool ltdb_key_is_record(TDB_DATA key)
note that the key for a record can depend on whether the
dn refers to a case sensitive index record or not
*/
-TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
+TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn)
{
- struct ldb_context *ldb = ldb_module_get_ctx(module);
TDB_DATA key;
char *key_str = NULL;
const char *dn_folded = NULL;
@@ -188,7 +188,7 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn)
goto failed;
}
- key_str = talloc_strdup(ldb, "DN=");
+ key_str = talloc_strdup(mem_ctx, "DN=");
if (!key_str) {
goto failed;
}
@@ -238,15 +238,16 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
/*
form a TDB_DATA for a record key
- caller frees
+ caller frees mem_ctx, which may or may not have the key
+ as a child.
- note that the key for a record can depend on whether the
- dn refers to a case sensitive index record or not
+ note that the key for a record can depend on whether a
+ GUID index is in use, or the DN is used as the key
*/
-TDB_DATA ltdb_key_msg(struct ldb_module *module,
+TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg)
{
- return ltdb_key_dn(module, msg->dn);
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
}
/*
@@ -336,16 +337,22 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
TDB_DATA tdb_key, tdb_data;
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
+ TALLOC_CTX *tdb_key_ctx = talloc_new(module);
+
+ if (tdb_key_ctx == NULL) {
+ return ldb_module_oom(module);
+ }
- tdb_key = ltdb_key_msg(module, msg);
+ tdb_key = ltdb_key_msg(module, tdb_key_ctx, msg);
if (tdb_key.dptr == NULL) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
ret = ldb_pack_data(ldb_module_get_ctx(module),
msg, &ldb_data);
if (ret == -1) {
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
@@ -370,7 +377,7 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
}
done:
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
talloc_free(ldb_data.data);
return ret;
@@ -531,14 +538,20 @@ int ltdb_delete_noindex(struct ldb_module *module,
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA tdb_key;
int ret;
+ TALLOC_CTX *tdb_key_ctx = talloc_new(module);
+
+ if (tdb_key_ctx == NULL) {
+ return ldb_module_oom(module);
+ }
- tdb_key = ltdb_key_dn(module, msg->dn);
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, msg->dn);
if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
}
ret = tdb_delete(ltdb->tdb, tdb_key);
- talloc_free(tdb_key.dptr);
+ TALLOC_FREE(tdb_key_ctx);
if (ret != 0) {
ret = ltdb_err_map(tdb_error(ltdb->tdb));
@@ -1187,13 +1200,13 @@ static int ltdb_rename(struct ltdb_context *ctx)
* Even in GUID index mode we use ltdb_key_dn() as we are
* trying to figure out if this is just a case rename
*/
- tdb_key = ltdb_key_dn(module, req->op.rename.newdn);
+ tdb_key = ltdb_key_dn(module, msg, req->op.rename.newdn);
if (!tdb_key.dptr) {
talloc_free(msg);
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key_old = ltdb_key_dn(module, req->op.rename.olddn);
+ tdb_key_old = ltdb_key_dn(module, msg, req->op.rename.olddn);
if (!tdb_key_old.dptr) {
talloc_free(msg);
talloc_free(tdb_key.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 1e639e9a60a..f79a8d0dd66 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -144,8 +144,10 @@ int ltdb_unlock_read(struct ldb_module *module);
* index, the old DN index and a possible future ID=
*/
bool ltdb_key_is_record(TDB_DATA key);
-TDB_DATA ltdb_key_dn(struct ldb_module *module, struct ldb_dn *dn);
-TDB_DATA ltdb_key_msg(struct ldb_module *module, const struct ldb_message *msg);
+TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn);
+TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
+ const struct ldb_message *msg);
TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
struct ltdb_private *ltdb,
TALLOC_CTX *mem_ctx,
--
2.11.0
From 9819022cc4e742c638fc3b38a1983ee9b2460321 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:53:34 +1200
Subject: [PATCH 38/88] ldb_tdb: Use ltdb_key_msg() in ltdb_delete_noindex()
This allows the optional use of GUID based TDB key.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f81fc054c3a..82a31cfbe82 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -544,7 +544,7 @@ int ltdb_delete_noindex(struct ldb_module *module,
return ldb_module_oom(module);
}
- tdb_key = ltdb_key_dn(module, tdb_key_ctx, msg->dn);
+ tdb_key = ltdb_key_msg(module, tdb_key_ctx, msg);
if (!tdb_key.dptr) {
TALLOC_FREE(tdb_key_ctx);
return LDB_ERR_OTHER;
--
2.11.0
From fc0928711807030c2602cf07f58434d17c4f0b0d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:11:13 +1200
Subject: [PATCH 39/88] ldb_tdb: Use the objectGUID (or similar) as the TDB key
in ltdb_key_msg()
When we have the full ldb_message we can read the objectGUID as the TDB key
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 25 ++++++++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 82a31cfbe82..70be65f00bd 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -247,7 +247,30 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg)
{
- return ltdb_key_dn(module, mem_ctx, msg->dn);
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ TDB_DATA key;
+ const struct ldb_val *guid_val;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
+ }
+
+ if (ldb_dn_is_special(msg->dn)) {
+ return ltdb_key_dn(module, mem_ctx, msg->dn);
+ }
+
+ guid_val = ldb_msg_find_ldb_val(msg,
+ ltdb->cache->GUID_index_attribute);
+ if (guid_val == NULL) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+
+ return ltdb_guid_to_key(module, ltdb, mem_ctx, guid_val);
+
}
/*
--
2.11.0
From c8410c11fd4536f714b500758710db9d93d48c5e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:12:30 +1200
Subject: [PATCH 40/88] ldb_tdb: Optionally use GUID index in ltdb_search_dn1()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 350f01b8093..249ecfeb810 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -271,11 +271,28 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return ldb_module_oom(module);
}
- /* form the key */
- tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
- if (!tdb_key.dptr) {
- TALLOC_FREE(tdb_key_ctx);
- return LDB_ERR_OPERATIONS_ERROR;
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
+ if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ } else if (ldb_dn_is_special(dn)) {
+ /* form the key */
+ tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
+ if (!tdb_key.dptr) {
+ TALLOC_FREE(tdb_key_ctx);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ } else {
+ /* Look in the index to find the key for this DN */
+ ret = ltdb_key_dn_from_idx(module, ltdb, tdb_key_ctx,
+ dn, &tdb_key);
+ if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(tdb_key_ctx);
+ return ret;
+ }
}
ret = ltdb_search_key(module, ltdb, tdb_key, msg, unpack_flags);
--
2.11.0
From 67da2651876b0d15227ed0cac170d7664269ba19 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 14 Aug 2017 16:27:46 +1200
Subject: [PATCH 41/88] ldb_tdb: Do not query an index on the
GUID_index_attribute
The objectGUID (or similar) is already the record key, there is
no need to index it to itself.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index f6dfb8fc3c9..3560aa64875 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -739,7 +739,32 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
dn_tree = *tree;
dn_tree.u.equality.attr = LTDB_IDXDN;
tree = &dn_tree;
+ } else if ((ltdb->cache->GUID_index_attribute != NULL) &&
+ (ldb_attr_cmp(tree->u.equality.attr,
+ ltdb->cache->GUID_index_attribute) == 0)) {
+ int ret;
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ list->dn = talloc_array(list, struct ldb_val, 1);
+ if (list->dn == NULL) {
+ ldb_module_oom(module);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ /*
+ * We need to go via the canonicalise_fn() to
+ * ensure we get the index in binary, rather
+ * than a string
+ */
+ ret = ltdb->GUID_index_syntax->canonicalise_fn(ldb,
+ list->dn,
+ &tree->u.equality.value,
+ &list->dn[0]);
+ if (ret != LDB_SUCCESS) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ list->count = 1;
+ return LDB_SUCCESS;
}
+
return ltdb_index_dn_simple(module, ltdb, tree, list);
}
--
2.11.0
From 3f7109a1b6add65950b1e45c458a93ce48e14ad0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 12:44:34 +1200
Subject: [PATCH 42/88] ldb_tdb: Do not add an index for GUID_index_attribute
This would be pointless and we no longer query for it.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 3560aa64875..90fd8994f83 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -624,6 +624,12 @@ static bool ltdb_is_indexed(struct ldb_module *module,
unsigned int i;
struct ldb_message_element *el;
+ if ((ltdb->cache->GUID_index_attribute != NULL) &&
+ (ldb_attr_cmp(attr,
+ ltdb->cache->GUID_index_attribute) == 0)) {
+ /* Implicity covered, this is the index key */
+ return false;
+ }
if (ldb->schema.index_handler_override) {
const struct ldb_schema_attribute *a
= ldb_schema_attribute_by_name(ldb, attr);
--
2.11.0
From c14bba02943cd5d251ffef6a142b0225ff48b274 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 17 Aug 2017 07:15:50 +1200
Subject: [PATCH 43/88] ldb_tdb: Add ltdb_idx_to_key() and use it in
ltdb_index_filter()
This will allow a common point to parse index records into a TDB key,
allowing them to be a GUID or DN in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 13 +++++++------
lib/ldb/ldb_tdb/ldb_tdb.c | 28 ++++++++++++++++++++++++++++
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++++
3 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 90fd8994f83..553b62e5da2 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1233,7 +1233,7 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
ldb = ldb_module_get_ctx(ac->module);
for (i = 0; i < dn_list->count; i++) {
- struct ldb_dn *dn;
+ TDB_DATA tdb_key;
int ret;
bool matched;
@@ -1242,16 +1242,17 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn = ldb_dn_from_ldb_val(msg, ldb, &dn_list->dn[i]);
- if (dn == NULL) {
- talloc_free(msg);
+ tdb_key = ltdb_idx_to_key(ac->module, ltdb,
+ ac, &dn_list->dn[i]);
+ if (tdb_key.dptr == NULL) {
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_search_dn1(ac->module, dn, msg,
+ ret = ltdb_search_key(ac->module, ltdb,
+ tdb_key, msg,
LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
- talloc_free(dn);
+ TALLOC_FREE(tdb_key.dptr);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* the record has disappeared? yes, this can happen */
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 70be65f00bd..bf3cddfbc1a 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -236,6 +236,34 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
return key;
}
+TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val)
+{
+ TDB_DATA key;
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+ struct ldb_dn *dn;
+
+ dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
+ if (dn == NULL) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ /* form the key */
+ key = ltdb_key_dn(module, mem_ctx, dn);
+ TALLOC_FREE(dn);
+ if (!key.dptr) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ return key;
+}
+
/*
form a TDB_DATA for a record key
caller frees mem_ctx, which may or may not have the key
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index f79a8d0dd66..c40d39fd5e4 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -152,6 +152,10 @@ TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
struct ltdb_private *ltdb,
TALLOC_CTX *mem_ctx,
const struct ldb_val *guid_val);
+TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From ddc4bc69242474656901810a861240127244d0b1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 09:59:54 +1200
Subject: [PATCH 44/88] ldb_tdb: Trust the BASE and ONELEVEL index
This avoids re-checking the fetched DN against the scope
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 28 ++++++++++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 553b62e5da2..8579b01ed33 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1265,8 +1265,18 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ldb_match_msg_error(ldb, msg,
- ac->tree, ac->base, ac->scope, &matched);
+ /* We trust the index for SCOPE_ONELEVEL and SCOPE_BASE */
+ if ((ac->scope == LDB_SCOPE_ONELEVEL
+ && ltdb->cache->one_level_indexes)
+ || ac->scope == LDB_SCOPE_BASE) {
+ ret = ldb_match_message(ldb, msg, ac->tree,
+ ac->scope, &matched);
+ } else {
+ ret = ldb_match_msg_error(ldb, msg,
+ ac->tree, ac->base,
+ ac->scope, &matched);
+ }
+
if (ret != LDB_SUCCESS) {
talloc_free(msg);
return ret;
@@ -1354,6 +1364,11 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
switch (ac->scope) {
case LDB_SCOPE_BASE:
+ /*
+ * If we ever start to also load the index values for
+ * the tree, we must ensure we strictly intersect with
+ * this list, as we trust the BASE index
+ */
ret = ltdb_index_dn_base_dn(ac->module, ltdb,
ac->base, dn_list);
if (ret != LDB_SUCCESS) {
@@ -1367,6 +1382,11 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
+ /*
+ * If we ever start to also load the index values for
+ * the tree, we must ensure we strictly intersect with
+ * this list, as we trust the ONELEVEL index
+ */
ret = ltdb_index_dn_one(ac->module, ltdb, ac->base, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
@@ -1380,6 +1400,10 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
+ /*
+ * Here we load the index for the tree. We have no
+ * index for the subtree.
+ */
ret = ltdb_index_dn(ac->module, ltdb, ac->tree, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
--
2.11.0
From 0d4339b0b78f8af18ee1572292a8b68aa8ad0e09 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 10 Aug 2017 17:05:37 +1200
Subject: [PATCH 45/88] ldb_tdb: Optionally use GUID index values a direct TDB
keys
This connects the GUID based index records to GUID based TDB keys.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index bf3cddfbc1a..292c181cee0 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -245,6 +245,10 @@ TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct ldb_dn *dn;
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return ltdb_guid_to_key(module, ltdb, mem_ctx, idx_val);
+ }
+
dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
if (dn == NULL) {
errno = EINVAL;
--
2.11.0
From 4b747e42c555a0727b7c8de7bdfd3d70d3d81701 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:57:57 +1200
Subject: [PATCH 46/88] ldb_tdb: Read from @INDEXLIST or an override if we are
using a GUID index
This allows all the previous patches to be enabled.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/common/ldb_attributes.c | 14 ++++++++++++++
lib/ldb/include/ldb_module.h | 16 ++++++++++++++++
lib/ldb/include/ldb_private.h | 3 +++
lib/ldb/ldb_tdb/ldb_cache.c | 10 ++++++++++
4 files changed, 43 insertions(+)
diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c
index 98ec5a409ba..32f25fd0fe8 100644
--- a/lib/ldb/common/ldb_attributes.c
+++ b/lib/ldb/common/ldb_attributes.c
@@ -395,3 +395,17 @@ void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
ldb->schema.index_handler_override = true;
ldb->schema.one_level_indexes = one_level_indexes;
}
+
+/*
+ * set that the GUID index mode is in operation
+ *
+ * The caller must ensure the supplied strings do not go out of
+ * scope (they are typically constant memory).
+ */
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component)
+{
+ ldb->schema.GUID_index_attribute = GUID_index_attribute;
+ ldb->schema.GUID_index_dn_component = GUID_index_dn_component;
+}
diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index 71b407496f0..ffa6c2eedb8 100644
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -183,6 +183,22 @@ void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
bool one_level_indexes);
+/**
+
+ \param ldb The ldb context
+ \param GUID_index_attribute The globally attribute (eg objectGUID)
+ on each entry
+ \param GUID_index_attribute The DN component matching the
+ globally attribute on each entry (eg GUID)
+
+ The caller must ensure the supplied strings do not go out of
+ scope (they are typically constant memory).
+
+*/
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component);
+
/* A useful function to build comparison functions with */
int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx,
ldb_attr_handler_t canonicalise_fn,
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index ab215411a9a..f999f7530bf 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -98,6 +98,9 @@ struct ldb_schema {
*/
bool index_handler_override;
bool one_level_indexes;
+
+ const char *GUID_index_attribute;
+ const char *GUID_index_dn_component;
};
/*
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 137ddf97fe8..7bf5f156b45 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -243,6 +243,10 @@ static int ltdb_index_load(struct ldb_module *module,
*/
ltdb->cache->attribute_indexes = true;
ltdb->cache->one_level_indexes = ldb->schema.one_level_indexes;
+ ltdb->cache->GUID_index_attribute
+ = ldb->schema.GUID_index_attribute;
+ ltdb->cache->GUID_index_dn_component
+ = ldb->schema.GUID_index_dn_component;
return 0;
}
@@ -276,6 +280,12 @@ static int ltdb_index_load(struct ldb_module *module,
if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
ltdb->cache->attribute_indexes = true;
}
+ ltdb->cache->GUID_index_attribute
+ = ldb_msg_find_attr_as_string(ltdb->cache->indexlist,
+ LTDB_IDXGUID, NULL);
+ ltdb->cache->GUID_index_dn_component
+ = ldb_msg_find_attr_as_string(ltdb->cache->indexlist,
+ LTDB_IDX_DN_GUID, NULL);
return 0;
}
--
2.11.0
From b8e55054d966a4fa29066ae0e2c1914ca2977f24 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 11:17:59 +1200
Subject: [PATCH 47/88] ldb_tdb: Add improved error strings on
ltdb_key_dn_from_idx() failure
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8579b01ed33..de91377e95f 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -316,6 +316,14 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
return LDB_ERR_NO_SUCH_OBJECT;
}
if (list->count > 1) {
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ __location__
+ ": Failed to read DN index "
+ "against %s for %s: too many "
+ "values (%u > 1)",
+ ltdb->cache->GUID_index_attribute,
+ dn_str, list->count);
return LDB_ERR_CONSTRAINT_VIOLATION;
}
--
2.11.0
From f04665f82d59adfd0298c91e8215ea9a7f04b48d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 12:59:50 +1200
Subject: [PATCH 48/88] ldb_tdb: Improve debugging in ltdb_modify_index_dn() on
casefold failure
This is unlikely, but when it happens it will be really painful to debug.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index de91377e95f..b77522c703c 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1632,6 +1632,15 @@ static int ltdb_modify_index_dn(struct ldb_module *module,
val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
if (val.data == NULL) {
+ const char *dn_str = ldb_dn_get_linearized(dn);
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ __location__
+ ": Failed to modify %s "
+ "against %s in %s: failed "
+ "to get casefold DN",
+ index,
+ ltdb->cache->GUID_index_attribute,
+ dn_str);
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From 304b513f20e0e7898858b2eb2c496b88c8e1506e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 14:24:44 +1200
Subject: [PATCH 49/88] ldb_tdb: Add unique index checking for @IDXDN
This will give us errors earlier if the index code becomes broken
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index b77522c703c..4e7b1012771 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1478,8 +1478,13 @@ static int ltdb_index_add1(struct ldb_module *module,
return ret;
}
+ /*
+ * Check for duplicates in unique indexes and for the @IDXDN
+ * DN -> GUID record
+ */
if (list->count > 0 &&
- a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX) {
+ (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
+ ldb_attr_cmp(el->name, LTDB_IDXDN) == 0)) {
/*
* We do not want to print info about a possibly
* confidential DN that the conflict was with in the
--
2.11.0
From e2e481f25acb120c5faa38e51b667282328df1b7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 12:37:51 +1200
Subject: [PATCH 50/88] ldb_tdb: Re-add of both existing DN and GUID must gives
LDB_ERR_ENTRY_ALREADY_EXISTS
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 292c181cee0..95b74bef391 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -525,6 +525,24 @@ static int ltdb_add_internal(struct ldb_module *module,
ret = ltdb_store(module, msg, TDB_INSERT);
if (ret != LDB_SUCCESS) {
+ /*
+ * Try really hard to get the right error code for
+ * a re-add situation, as this can matter!
+ */
+ if (ret == LDB_ERR_CONSTRAINT_VIOLATION) {
+ int ret2;
+ struct ldb_dn *dn2 = NULL;
+ TALLOC_CTX *mem_ctx = talloc_new(module);
+ if (mem_ctx == NULL) {
+ return ldb_module_operr(module);
+ }
+ ret2 = ltdb_search_base(module, module,
+ msg->dn, &dn2);
+ TALLOC_FREE(mem_ctx);
+ if (ret2 == LDB_SUCCESS) {
+ ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
+ }
+ }
if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
ldb_asprintf_errstring(ldb,
"Entry %s already exists",
--
2.11.0
From ee032691c38622fc0a98d702acc91ea2e9ec8e28 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 14:25:46 +1200
Subject: [PATCH 51/88] ldb_tdb: Change error code on unique index violation
The LDB_ERR_ENTRY_ALREADY_EXISTS error code is detected in repl_meta_data as indicating
that the DN exists, and that a conflict record should be created.
This is really a constraint violation, not a duplicate record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 4e7b1012771..054bb8f18e9 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1502,7 +1502,7 @@ static int ltdb_index_add1(struct ldb_module *module,
el->name,
ldb_dn_get_linearized(msg->dn));
talloc_free(list);
- return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ return LDB_ERR_CONSTRAINT_VIOLATION;
}
/* overallocate the list a bit, to reduce the number of
--
2.11.0
From ecdc068d6de787c6cb58738f37485edaee8e9ce6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 15:35:32 +1200
Subject: [PATCH 52/88] ldb_tdb: sort GUID index list at add time by inserting
in sorted order
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 154 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 152 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 054bb8f18e9..055cf38fb6e 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1,4 +1,122 @@
/*
+ Unix SMB/CIFS implementation.
+
+ a generic binary search macro
+
+ Copyright (C) Andrew Tridgell 2009
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _BINSEARCH_H
+#define _BINSEARCH_H
+
+/* a binary array search, where the array is an array of pointers to structures,
+ and we want to find a match for 'target' on 'field' in those structures.
+
+ Inputs:
+ array: base pointer to an array of structures
+ arrray_size: number of elements in the array
+ field: the name of the field in the structure we are keying off
+ target: the field value we are looking for
+ comparison_fn: the comparison function
+ result: where the result of the search is put
+
+ if the element is found, then 'result' is set to point to the found array element. If not,
+ then 'result' is set to NULL.
+
+ The array is assumed to be sorted by the same comparison_fn as the
+ search (with, for example, qsort)
+ */
+#define BINARY_ARRAY_SEARCH_P(array, array_size, field, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i]->field); \
+ if (_r == 0) { (result) = array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+/*
+ like BINARY_ARRAY_SEARCH_P, but assumes that the array is an array
+ of structures, rather than pointers to structures
+
+ result points to the found structure, or NULL
+ */
+#define BINARY_ARRAY_SEARCH(array, array_size, field, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i].field); \
+ if (_r == 0) { (result) = &array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+/*
+ like BINARY_ARRAY_SEARCH_P, but assumes that the array is an array
+ of elements, rather than pointers to structures
+
+ result points to the found structure, or NULL
+ */
+#define BINARY_ARRAY_SEARCH_V(array, array_size, target, comparison_fn, result) do { \
+ int32_t _b, _e; \
+ (result) = NULL; \
+ if (array_size) { for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b+_e)/2; \
+ int _r = comparison_fn(target, array[_i]); \
+ if (_r == 0) { (result) = &array[_i]; break; } \
+ if (_r < 0) _e = _i - 1; else _b = _i + 1; \
+ }} } while (0)
+
+
+/*
+ like BINARY_ARRAY_SEARCH_V, but if an exact result is not found, the 'next'
+ argument will point to the element after the place where the exact result
+ would have been. If an exact result is found, 'next' will be NULL. If the
+ target is beyond the end of the list, both 'exact' and 'next' will be NULL.
+ Unlike other binsearch macros, where there are several elements that compare
+ the same, the exact result will always point to the first one.
+
+ If you don't care to distinguish between the 'greater than' and 'equals'
+ cases, you can use the same pointer for both 'exact' and 'next'.
+
+ As with all the binsearch macros, the comparison function is always called
+ with the search term first.
+ */
+#define BINARY_ARRAY_SEARCH_GTE(array, array_size, target, comparison_fn, \
+ exact, next) do { \
+ int32_t _b, _e; \
+ (exact) = NULL; (next) = NULL; \
+ if ((array_size) > 0) { \
+ for (_b = 0, _e = (array_size)-1; _b <= _e; ) { \
+ int32_t _i = (_b + _e) / 2; \
+ int _r = comparison_fn(target, &array[_i]); \
+ if (_r == 0) { \
+ (exact) = &array[_i]; \
+ _e = _i - 1; \
+ } else if (_r < 0) { _e = _i - 1; \
+ } else { _b = _i + 1; } \
+ } \
+ if ((exact) == NULL &&_b < (array_size)) { \
+ (next) = &array[_b]; \
+ } } } while (0)
+
+#endif
+
+/*
ldb database library
Copyright (C) Andrew Tridgell 2004-2009
@@ -87,6 +205,22 @@ static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1,
return memcmp(v1->data, v2->data, v1->length);
}
+/*
+ see if two ldb_val structures contain exactly the same data
+ return -1 or 1 for a mismatch, 0 for match
+*/
+static int ldb_val_equal_exact_ordered(const struct ldb_val v1,
+ const struct ldb_val *v2)
+{
+ if (v1.length > v2->length) {
+ return -1;
+ }
+ if (v1.length < v2->length) {
+ return 1;
+ }
+ return memcmp(v1.data, v2->data, v1.length);
+}
+
/*
find a entry in a dn_list, using a ldb_val. Uses a case sensitive
@@ -1525,6 +1659,7 @@ static int ltdb_index_add1(struct ldb_module *module,
list->dn[list->count].length = strlen(dn_str);
} else {
const struct ldb_val *key_val;
+ struct ldb_val *exact = NULL, *next = NULL;
key_val = ldb_msg_find_ldb_val(msg,
ltdb->cache->GUID_index_attribute);
if (key_val == NULL) {
@@ -1536,8 +1671,23 @@ static int ltdb_index_add1(struct ldb_module *module,
talloc_free(list);
return ldb_module_operr(module);
}
- list->dn[list->count] = ldb_val_dup(list->dn, key_val);
- if (list->dn[list->count].data == NULL) {
+
+ BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
+ *key_val, ldb_val_equal_exact_ordered,
+ exact, next);
+ if (exact != NULL) {
+ talloc_free(list);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ if (next == NULL) {
+ next = &list->dn[list->count];
+ } else {
+ memmove(&next[1], next,
+ sizeof(*next) * (list->count - (next - list->dn)));
+ }
+ *next = ldb_val_dup(list->dn, key_val);
+ if (next->data == NULL) {
talloc_free(list);
return ldb_module_operr(module);
}
--
2.11.0
From ce7ecb0c3980dcd95e7c3f03c7aaf31840ecc6df Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:51:30 +1200
Subject: [PATCH 53/88] ldb_tdb: Sort inputs to list_union()
This allows us to merge the lists finding common values.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 38 +++++++++++++++++++++++++++++++++++---
1 file changed, 35 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 055cf38fb6e..a14181513d3 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -170,6 +170,9 @@ static int ltdb_index_dn_base_dn(struct ldb_module *module,
struct ldb_dn *base_dn,
struct dn_list *dn_list);
+static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
+ struct dn_list *list);
+
/* we put a @IDXVERSION attribute on index entries. This
allows us to tell if it was written by an older version
*/
@@ -852,7 +855,9 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
}
-static bool list_union(struct ldb_context *, struct dn_list *, const struct dn_list *);
+static bool list_union(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
+ struct dn_list *list, struct dn_list *list2);
/*
return a list of dn's that might match a leaf indexed search
@@ -990,7 +995,8 @@ static bool list_intersect(struct ldb_context *ldb,
list = list | list2
*/
static bool list_union(struct ldb_context *ldb,
- struct dn_list *list, const struct dn_list *list2)
+ struct ltdb_private *ltdb,
+ struct dn_list *list, struct dn_list *list2)
{
struct ldb_val *dn3;
@@ -1011,6 +1017,13 @@ static bool list_union(struct ldb_context *ldb,
return true;
}
+ /*
+ * Sort the lists (if not in GUID DN mode) so we can do
+ * the de-duplication during the merge
+ */
+ ltdb_dn_list_sort(ltdb, list);
+ ltdb_dn_list_sort(ltdb, list2);
+
dn3 = talloc_array(list, struct ldb_val, list->count + list2->count);
if (!dn3) {
ldb_oom(ldb);
@@ -1073,7 +1086,7 @@ static int ltdb_index_dn_or(struct ldb_module *module,
return ret;
}
- if (!list_union(ldb, list, list2)) {
+ if (!list_union(ldb, ltdb, list, list2)) {
talloc_free(list2);
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1453,6 +1466,25 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
}
/*
+ sort a DN list
+ */
+static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
+ struct dn_list *list)
+{
+ if (list->count < 2) {
+ return;
+ }
+
+ /* We know the list is sorted when using the GUID index */
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ return;
+ }
+
+ TYPESAFE_QSORT(list->dn, list->count,
+ ldb_val_equal_exact_for_qsort);
+}
+
+/*
remove any duplicated entries in a indexed result
*/
static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
--
2.11.0
From cc68d9fdb1c3427ad8645734883760a072c302a3 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:07:45 +1200
Subject: [PATCH 54/88] ldb_tdb: Rework list_union to not return duplicates,
and keep sort order
This allows the binary search to still operate on the list, even after
a or operator in the search expression
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 35 +++++++++++++++++++++++++++++++----
1 file changed, 31 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index a14181513d3..387898304cc 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -999,6 +999,7 @@ static bool list_union(struct ldb_context *ldb,
struct dn_list *list, struct dn_list *list2)
{
struct ldb_val *dn3;
+ unsigned int i = 0, j = 0, k = 0;
if (list2->count == 0) {
/* X | 0 == X */
@@ -1030,12 +1031,38 @@ static bool list_union(struct ldb_context *ldb,
return false;
}
- /* we allow for duplicates here, and get rid of them later */
- memcpy(dn3, list->dn, sizeof(list->dn[0])*list->count);
- memcpy(dn3+list->count, list2->dn, sizeof(list2->dn[0])*list2->count);
+ while (i < list->count || j < list2->count) {
+ int cmp;
+ if (i >= list->count) {
+ cmp = 1;
+ } else if (j >= list2->count) {
+ cmp = -1;
+ } else {
+ cmp = ldb_val_equal_exact_ordered(list->dn[i],
+ &list2->dn[j]);
+ }
+
+ if (cmp < 0) {
+ /* Take list */
+ dn3[k] = list->dn[i];
+ i++;
+ k++;
+ } else if (cmp > 0) {
+ /* Take list2 */
+ dn3[k] = list2->dn[j];
+ j++;
+ k++;
+ } else {
+ /* Equal, take list */
+ dn3[k] = list->dn[i];
+ i++;
+ j++;
+ k++;
+ }
+ }
list->dn = dn3;
- list->count += list2->count;
+ list->count = k;
return true;
}
--
2.11.0
From 4fbba7886aa43f0e7632bf3fec8b40d3842212ea Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 21 Aug 2017 15:51:19 +1200
Subject: [PATCH 55/88] ldb_tdb: Use a binary search to speed up
ltdb_dn_list_find_val()
This only works if we have the GUID index format, as otherwise these are unsorted.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 26 ++++++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 387898304cc..9d85494f4d0 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -236,12 +236,30 @@ static int ltdb_dn_list_find_val(struct ltdb_private *ltdb,
const struct ldb_val *v)
{
unsigned int i;
- for (i=0; i<list->count; i++) {
- if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
- return i;
+ struct ldb_val *exact = NULL, *next = NULL;
+
+ if (ltdb->cache->GUID_index_attribute == NULL) {
+ for (i=0; i<list->count; i++) {
+ if (ldb_val_equal_exact(&list->dn[i], v) == 1) {
+ return i;
+ }
}
+ return -1;
+ }
+
+ BINARY_ARRAY_SEARCH_GTE(list->dn, list->count,
+ *v, ldb_val_equal_exact_ordered,
+ exact, next);
+ if (exact == NULL) {
+ return -1;
}
- return -1;
+ /* Not required, but keeps the compiler quiet */
+ if (next != NULL) {
+ return -1;
+ }
+
+ i = exact - list->dn;
+ return i;
}
/*
--
2.11.0
From d8d0c0cac1e41a29c8cb428ddf24d6957d98bded Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:16:56 +1200
Subject: [PATCH 56/88] ldb_tdb: Use the binary search more efficiently in
list_intersect()
This change ensures we walk the short list and look up into the longer of the two lists.
ltdb_dn_list_find_val() will do a binary search for the GUID case.
Before GUID indexes this was O(n*m), now it is O(n*log(m)).
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 9d85494f4d0..0a9e1167dd5 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -948,6 +948,7 @@ static bool list_intersect(struct ldb_context *ldb,
struct ltdb_private *ltdb,
struct dn_list *list, const struct dn_list *list2)
{
+ const struct dn_list *short_list, *long_list;
struct dn_list *list3;
unsigned int i;
@@ -980,6 +981,14 @@ static bool list_intersect(struct ldb_context *ldb,
return true;
}
+ if (list->count > list2->count) {
+ short_list = list2;
+ long_list = list;
+ } else {
+ short_list = list;
+ long_list = list2;
+ }
+
list3 = talloc_zero(list, struct dn_list);
if (list3 == NULL) {
return false;
@@ -992,10 +1001,11 @@ static bool list_intersect(struct ldb_context *ldb,
}
list3->count = 0;
- for (i=0;i<list->count;i++) {
- if (ltdb_dn_list_find_val(ltdb, list2,
- &list->dn[i]) != -1) {
- list3->dn[list3->count] = list->dn[i];
+ for (i=0;i<short_list->count;i++) {
+ /* For the GUID index case, this is a binary search */
+ if (ltdb_dn_list_find_val(ltdb, long_list,
+ &short_list->dn[i]) != -1) {
+ list3->dn[list3->count] = short_list->dn[i];
list3->count++;
}
}
--
2.11.0
From 11564aa31e144f1deb17117ffa228dc6defd1424 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:17:56 +1200
Subject: [PATCH 57/88] ldb_tdb: Reduce memory consumption in list_intersect()
We will never have more results than is in either list or list2.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 0a9e1167dd5..0418988b88d 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -994,7 +994,8 @@ static bool list_intersect(struct ldb_context *ldb,
return false;
}
- list3->dn = talloc_array(list3, struct ldb_val, list->count);
+ list3->dn = talloc_array(list3, struct ldb_val,
+ MIN(list->count, list2->count));
if (!list3->dn) {
talloc_free(list3);
return false;
--
2.11.0
From 98917e9184cb1e41c122d623a0723b94fe2ac3ac Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 22 Aug 2017 11:55:01 +1200
Subject: [PATCH 58/88] ldb_tdb: Duplicate values are no longer permitted in
the index
By removing the qsort() we avoid work.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 29 -----------------------------
1 file changed, 29 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 0418988b88d..6043b16a6ad 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1541,34 +1541,6 @@ static void ltdb_dn_list_sort(struct ltdb_private *ltdb,
}
/*
- remove any duplicated entries in a indexed result
- */
-static void ltdb_dn_list_remove_duplicates(struct dn_list *list)
-{
- unsigned int i, new_count;
-
- if (list->count < 2) {
- return;
- }
-
- TYPESAFE_QSORT(list->dn, list->count,
- ldb_val_equal_exact_for_qsort);
-
- new_count = 1;
- for (i=1; i<list->count; i++) {
- if (ldb_val_equal_exact(&list->dn[i],
- &list->dn[new_count-1]) == 0) {
- if (new_count != i) {
- list->dn[new_count] = list->dn[i];
- }
- new_count++;
- }
- }
-
- list->count = new_count;
-}
-
-/*
search the database with a LDAP-like expression using indexes
returns -1 if an indexed search is not possible, in which
case the caller should call ltdb_search_full()
@@ -1639,7 +1611,6 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return ret;
}
- ltdb_dn_list_remove_duplicates(dn_list);
break;
}
--
2.11.0
From e6374c8c41f6d00dd33af94cd2d814d6a400333f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 12:06:48 +1200
Subject: [PATCH 59/88] ldb: Add an unused objectGUID to each record in
SearchTests
This will then be used by the GUID index tests.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 81 +++++++++++++++++++++++++++++++--------------
1 file changed, 56 insertions(+), 25 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index b95266b0384..1db2c5ca003 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -616,79 +616,110 @@ class SearchTests(TestCase):
self.filename = os.path.join(self.testdir, "search_test.ldb")
self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
- self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
+ # Note that we can't use the name objectGUID here, as we
+ # want to stay clear of the objectGUID handler in LDB and
+ # instead use just the 16 bytes raw, which we just keep
+ # to printable chars here for ease of handling.
+
+ self.l.add({"dn": "DC=SAMBA,DC=ORG",
+ "name": b"samba.org",
+ "objectUUID": b"0123456789abcddf"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
"name": b"Admins",
- "x": "z", "y": "a"})
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG",
"name": b"Users",
- "x": "z", "y": "a"})
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG",
"name": b"OU #1",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde3"})
self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG",
"name": b"OU #2",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde4"})
self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG",
"name": b"OU #3",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde5"})
self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG",
"name": b"OU #4",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde6"})
self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG",
"name": b"OU #5",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde7"})
self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG",
"name": b"OU #6",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde8"})
self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG",
"name": b"OU #7",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde9"})
self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG",
"name": b"OU #8",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcde0"})
self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG",
"name": b"OU #9",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdea"})
self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdeb"})
self.l.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "a"})
+ "x": "y", "y": "a",
+ "objectUUID": b"0123456789abcdec"})
self.l.add({"dn": "OU=OU12,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "y", "y": "b"})
+ "x": "y", "y": "b",
+ "objectUUID": b"0123456789abcded"})
self.l.add({"dn": "OU=OU13,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcdee"})
self.l.add({"dn": "OU=OU14,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd01"})
self.l.add({"dn": "OU=OU15,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd02"})
self.l.add({"dn": "OU=OU16,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd03"})
self.l.add({"dn": "OU=OU17,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd04"})
self.l.add({"dn": "OU=OU18,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd05"})
self.l.add({"dn": "OU=OU19,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd06"})
self.l.add({"dn": "OU=OU20,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "b"})
+ "x": "x", "y": "b",
+ "objectUUID": b"0123456789abcd07"})
self.l.add({"dn": "OU=OU21,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "c"})
+ "x": "x", "y": "c",
+ "objectUUID": b"0123456789abcd08"})
self.l.add({"dn": "OU=OU22,DC=SAMBA,DC=ORG",
"name": b"OU #10",
- "x": "x", "y": "c"})
+ "x": "x", "y": "c",
+ "objectUUID": b"0123456789abcd09"})
def test_base(self):
"""Testing a search"""
--
2.11.0
From c959899a531ea3284ffc7e60c293f1abcb814508 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 15:34:31 +1200
Subject: [PATCH 60/88] ldb: Also test the new GUID index mode
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 1db2c5ca003..cc707edab3e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -934,6 +934,17 @@ class IndexedSearchTests(SearchTests):
"@IDXATTR": [b"x", b"y", b"ou"],
"@IDXONE": [b"1"]})
+class GUIDIndexedSearchTests(SearchTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedSearchTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]})
+
class DnTests(TestCase):
--
2.11.0
From 1d6ac59e250fa3dd729fc9aeceb0c45b7a47e6ba Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 22:22:27 +1200
Subject: [PATCH 61/88] ldb_tdb: Add tests for add/modify with the GUID index
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 84 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index cc707edab3e..bbdb775ceb3 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -945,6 +945,90 @@ class GUIDIndexedSearchTests(SearchTests):
"@IDXGUID": [b"objectUUID"],
"@IDX_DN_GUID": [b"GUID"]})
+class AddModifyTests(TestCase):
+ def tearDown(self):
+ shutil.rmtree(self.testdir)
+ super(AddModifyTests, self).tearDown()
+
+ def setUp(self):
+ super(AddModifyTests, self).setUp()
+ self.testdir = tempdir()
+ self.filename = os.path.join(self.testdir, "add_test.ldb")
+ self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l.add({"dn": "DC=SAMBA,DC=ORG",
+ "name": b"samba.org",
+ "objectUUID": b"0123456789abcdef"})
+
+ def test_add_dup(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ try:
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ self.fail("Should have failed adding dupliate entry")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ def test_add_del_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.delete("OU=DUP,DC=SAMBA,DC=ORG")
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+ def test_add_move_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+
+class IndexedAddModifyTests(AddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(IndexedAddModifyTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"]})
+
+class GUIDIndexedAddModifyTests(AddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedAddModifyTests, self).setUp()
+ self.l.add({"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]})
+
+ def test_duplicate_GUID(self):
+ try:
+ self.l.add({"dn": "OU=DUPGUID,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcdef"})
+ self.fail("Should have failed adding dupliate GUID")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+
class DnTests(TestCase):
--
2.11.0
From 86c1639cf5df4ad0ca9ecdf57a015c58d78b1006 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 28 Aug 2017 15:37:28 +1200
Subject: [PATCH 62/88] ldb_tdb: Use the DN extracted from the DB to filter the
message later
This should ensure that the upper or lower case the user chooses does not impact
on the filtering, at least for database that have checkBaseOnSearch set.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 38 +++++++++++++++++++++++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.c | 6 ++++--
lib/ldb/ldb_tdb/ldb_tdb.h | 5 ++++-
3 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 249ecfeb810..761d33ed6d6 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -113,7 +113,10 @@ static int msg_add_distinguished_name(struct ldb_message *msg)
return LDB_ERR_NO_SUCH_OBJECT on record-not-found
and LDB_SUCCESS on success
*/
-int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
+int ltdb_search_base(struct ldb_module *module,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ struct ldb_dn **ret_dn)
{
int exists;
int ret;
@@ -137,16 +140,34 @@ int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn)
ret = ltdb_search_dn1(module, dn,
msg,
- LDB_UNPACK_DATA_FLAG_NO_DN|
LDB_UNPACK_DATA_FLAG_NO_ATTRS);
- talloc_free(msg);
if (ret == LDB_SUCCESS) {
+ const char *dn_linearized
+ = ldb_dn_get_linearized(dn);
+ const char *msg_dn_linearlized
+ = ldb_dn_get_linearized(msg->dn);
+
+ if (strcmp(dn_linearized, msg_dn_linearlized) == 0) {
+ /*
+ * Re-use the full incoming DN for
+ * subtree checks
+ */
+ *ret_dn = dn;
+ } else {
+ /*
+ * Use the string DN from the unpack, so that
+ * we have a case-exact match of the base
+ */
+ *ret_dn = talloc_steal(mem_ctx, msg->dn);
+ }
exists = true;
} else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
exists = false;
} else {
+ talloc_free(msg);
return ret;
}
+ talloc_free(msg);
if (exists) {
return LDB_SUCCESS;
}
@@ -721,8 +742,15 @@ int ltdb_search(struct ltdb_context *ctx)
return ret;
} else if (ltdb->check_base) {
- /* This database has been marked as 'checkBaseOnSearch', so do a spot check of the base dn */
- ret = ltdb_search_base(module, req->op.search.base);
+ /*
+ * This database has been marked as
+ * 'checkBaseOnSearch', so do a spot check of the base
+ * dn. Also optimise the subsequent filter by filling
+ * in the ctx->base to be exactly case correct
+ */
+ ret = ltdb_search_base(module, ctx,
+ req->op.search.base,
+ &ctx->base);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
ldb_asprintf_errstring(ldb,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 95b74bef391..589050642e6 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1246,6 +1246,7 @@ static int ltdb_rename(struct ltdb_context *ctx)
struct ldb_message *msg;
int ret = LDB_SUCCESS;
TDB_DATA tdb_key, tdb_key_old;
+ struct ldb_dn *db_dn;
ldb_request_set_state(req, LDB_ASYNC_PENDING);
@@ -1292,8 +1293,9 @@ static int ltdb_rename(struct ltdb_context *ctx)
*/
if (tdb_key_old.dsize != tdb_key.dsize
|| memcmp(tdb_key.dptr, tdb_key_old.dptr, tdb_key.dsize) != 0) {
- ret = ltdb_search_base(module,
- req->op.rename.newdn);
+ ret = ltdb_search_base(module, msg,
+ req->op.rename.newdn,
+ &db_dn);
if (ret == LDB_SUCCESS) {
ret = LDB_ERR_ENTRY_ALREADY_EXISTS;
} else if (ret == LDB_ERR_NO_SUCH_OBJECT) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index c40d39fd5e4..0e6e9797d4b 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -126,7 +126,10 @@ int ltdb_has_wildcard(struct ldb_module *module, const char *attr_name,
void ltdb_search_dn1_free(struct ldb_module *module, struct ldb_message *msg);
int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_message *msg,
unsigned int unpack_flags);
-int ltdb_search_base(struct ldb_module *module, struct ldb_dn *dn);
+int ltdb_search_base(struct ldb_module *module,
+ TALLOC_CTX *mem_ctx,
+ struct ldb_dn *dn,
+ struct ldb_dn **ret_dn);
int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
struct TDB_DATA tdb_key,
struct ldb_message *msg,
--
2.11.0
From 3d3e7b56e59f48593a9f600670fe26459f0d1b0f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 29 Aug 2017 10:40:22 +1200
Subject: [PATCH 63/88] ldb: Add tests for base DN of a different case
This ensures we cover the case where the DN does not match the DB exactly
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index bbdb775ceb3..4c35e44ee28 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -616,6 +616,9 @@ class SearchTests(TestCase):
self.filename = os.path.join(self.testdir, "search_test.ldb")
self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l.add({"dn": "@ATTRIBUTES",
+ "DC": "CASE_INSENSITIVE"})
+
# Note that we can't use the name objectGUID here, as we
# want to stay clear of the objectGUID handler in LDB and
# instead use just the 16 bytes raw, which we just keep
@@ -728,6 +731,13 @@ class SearchTests(TestCase):
scope=ldb.SCOPE_BASE)
self.assertEqual(len(res11), 1)
+ def test_base_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="OU=OU11,DC=samba,DC=org",
+ scope=ldb.SCOPE_BASE)
+ self.assertEqual(len(res11), 1)
+
def test_base_or(self):
"""Testing a search"""
@@ -804,6 +814,14 @@ class SearchTests(TestCase):
expression="(&(x=y)(|(y=b)(y=c)))")
self.assertEqual(len(res11), 1)
+ def test_subtree_and2_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="DC=samba,DC=org",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(&(x=y)(|(y=b)(y=c)))")
+ self.assertEqual(len(res11), 1)
+
def test_subtree_or(self):
"""Testing a search"""
@@ -860,6 +878,14 @@ class SearchTests(TestCase):
expression="(|(x=y)(y=b))")
self.assertEqual(len(res11), 20)
+ def test_one_or2_lower(self):
+ """Testing a search"""
+
+ res11 = self.l.search(base="DC=samba,DC=org",
+ scope=ldb.SCOPE_ONELEVEL,
+ expression="(|(x=y)(y=b))")
+ self.assertEqual(len(res11), 20)
+
def test_subtree_and_or(self):
"""Testing a search"""
--
2.11.0
From 45c03c2058a087c1c1c84b471b136a798173f0aa Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 30 Aug 2017 13:18:20 +1200
Subject: [PATCH 64/88] ldb_tdb: Optimise ltdb_search_and_return_base() to
re-use casefolding
The casefolding of a DN is one of the more expensive and pointless things in LDB
operation. The ldb_dn abstraction works hard to avoid duplicating this work, but
we can work harder to save that information.
Here we copy in the DN, that has been casefolded already for the index,
and keep that as the returned DN, after stripping any extended components.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_search.c | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 761d33ed6d6..07757f86f3b 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -593,6 +593,8 @@ static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
{
struct ldb_message *msg, *filtered_msg;
struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
+ const char *dn_linearized;
+ const char *msg_dn_linearlized;
int ret;
bool matched;
@@ -642,9 +644,32 @@ static int ltdb_search_and_return_base(struct ltdb_private *ltdb,
return LDB_SUCCESS;
}
- /* filter the attributes that the user wants */
+ dn_linearized = ldb_dn_get_linearized(ctx->base);
+ msg_dn_linearlized = ldb_dn_get_linearized(msg->dn);
+
+ if (strcmp(dn_linearized, msg_dn_linearlized) == 0) {
+ /*
+ * If the DN is exactly the same string, then
+ * re-use the full incoming DN for the
+ * returned result, as it has already been
+ * casefolded
+ */
+ msg->dn = ctx->base;
+ }
+
+ /*
+ * filter the attributes that the user wants.
+ *
+ * This copies msg->dn including the casefolding, so the above
+ * assignment is safe
+ */
ret = ltdb_filter_attrs(ctx, msg, ctx->attrs, &filtered_msg);
+ /*
+ * Remove any extended components possibly copied in from
+ * msg->dn, we just want the casefold components
+ */
+ ldb_dn_remove_extended_components(filtered_msg->dn);
talloc_free(msg);
if (ret == -1) {
--
2.11.0
From f184e89dddec102fccb65d09f94eaa468c7417ff Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:04:43 +1200
Subject: [PATCH 65/88] ldb_tdb: Move constants into ldb_tdb.h
This helps ensure we keep these all in sync.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 6043b16a6ad..8f558195e10 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -408,7 +408,6 @@ normal_index:
list->count = el->num_values;
} else {
unsigned int i;
- static const size_t GUID_val_size = 16;
if (version != LTDB_GUID_INDEXING_VERSION) {
/* This is quite likely during the DB startup
on first upgrade to using a GUID index */
@@ -425,11 +424,11 @@ normal_index:
return LDB_ERR_OPERATIONS_ERROR;
}
- if ((el->values[0].length % GUID_val_size) != 0) {
+ if ((el->values[0].length % LTDB_GUID_SIZE) != 0) {
return LDB_ERR_OPERATIONS_ERROR;
}
- list->count = el->values[0].length / GUID_val_size;
+ list->count = el->values[0].length / LTDB_GUID_SIZE;
list->dn = talloc_array(list, struct ldb_val, list->count);
/*
@@ -438,8 +437,9 @@ normal_index:
*/
talloc_steal(list->dn, msg);
for (i = 0; i < list->count; i++) {
- list->dn[i].data = &el->values[0].data[i * GUID_val_size];
- list->dn[i].length = GUID_val_size;
+ list->dn[i].data
+ = &el->values[0].data[i * LTDB_GUID_SIZE];
+ list->dn[i].length = LTDB_GUID_SIZE;
}
}
--
2.11.0
From 0032d93f21c7883eb2b3068014b84f53899121fc Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:06:15 +1200
Subject: [PATCH 66/88] ldb_tdb: Avoid allocation of a DN between the GUID
index and the DB lookup
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 28 +++++++++-----
lib/ldb/ldb_tdb/ldb_search.c | 34 ++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.c | 90 +++++++++++++++++++++++++-------------------
lib/ldb/ldb_tdb/ldb_tdb.h | 18 +++++----
4 files changed, 106 insertions(+), 64 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8f558195e10..78675e8ae56 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -479,12 +479,15 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
"values (%u > 1)",
ltdb->cache->GUID_index_attribute,
dn_str, list->count);
+ TALLOC_FREE(list);
return LDB_ERR_CONSTRAINT_VIOLATION;
}
- *tdb_key = ltdb_guid_to_key(module, ltdb,
- mem_ctx, &list->dn[0]);
- if (tdb_key->dptr == NULL) {
+ /* The tdb_key memory is allocated by the caller */
+ ret = ltdb_guid_to_key(module, ltdb,
+ &list->dn[0], tdb_key);
+
+ if (ret != LDB_SUCCESS) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1444,7 +1447,11 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
ldb = ldb_module_get_ctx(ac->module);
for (i = 0; i < dn_list->count; i++) {
- TDB_DATA tdb_key;
+ uint8_t guid_key[LTDB_GUID_KEY_SIZE];
+ TDB_DATA tdb_key = {
+ .dptr = guid_key,
+ .dsize = sizeof(guid_key)
+ };
int ret;
bool matched;
@@ -1453,17 +1460,20 @@ static int ltdb_index_filter(struct ltdb_private *ltdb,
return LDB_ERR_OPERATIONS_ERROR;
}
- tdb_key = ltdb_idx_to_key(ac->module, ltdb,
- ac, &dn_list->dn[i]);
- if (tdb_key.dptr == NULL) {
- return LDB_ERR_OPERATIONS_ERROR;
+ ret = ltdb_idx_to_key(ac->module, ltdb,
+ ac, &dn_list->dn[i],
+ &tdb_key);
+ if (ret != LDB_SUCCESS) {
+ return ret;
}
ret = ltdb_search_key(ac->module, ltdb,
tdb_key, msg,
LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC|
LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC);
- TALLOC_FREE(tdb_key.dptr);
+ if (tdb_key.dptr != guid_key) {
+ TALLOC_FREE(tdb_key.dptr);
+ }
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* the record has disappeared? yes, this can happen */
talloc_free(msg);
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 07757f86f3b..0af230f219b 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -285,14 +285,19 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret;
- TDB_DATA tdb_key;
-
- TALLOC_CTX *tdb_key_ctx = talloc_new(msg);
- if (!tdb_key_ctx) {
- return ldb_module_oom(module);
- }
+ uint8_t guid_key[LTDB_GUID_KEY_SIZE];
+ TDB_DATA tdb_key = {
+ .dptr = guid_key,
+ .dsize = sizeof(guid_key)
+ };
+ TALLOC_CTX *tdb_key_ctx = NULL;
if (ltdb->cache->GUID_index_attribute == NULL) {
+ tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
@@ -300,6 +305,11 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return LDB_ERR_OPERATIONS_ERROR;
}
} else if (ldb_dn_is_special(dn)) {
+ tdb_key_ctx = talloc_new(msg);
+ if (!tdb_key_ctx) {
+ return ldb_module_oom(module);
+ }
+
/* form the key */
tdb_key = ltdb_key_dn(module, tdb_key_ctx, dn);
if (!tdb_key.dptr) {
@@ -307,11 +317,17 @@ int ltdb_search_dn1(struct ldb_module *module, struct ldb_dn *dn, struct ldb_mes
return LDB_ERR_OPERATIONS_ERROR;
}
} else {
- /* Look in the index to find the key for this DN */
- ret = ltdb_key_dn_from_idx(module, ltdb, tdb_key_ctx,
+ /*
+ * Look in the index to find the key for this DN.
+ *
+ * the tdb_key memory is allocated above, msg is just
+ * used for internal memory.
+ *
+ */
+ ret = ltdb_key_dn_from_idx(module, ltdb,
+ msg,
dn, &tdb_key);
if (ret != LDB_SUCCESS) {
- TALLOC_FREE(tdb_key_ctx);
return ret;
}
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 589050642e6..b4ce2470352 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -210,62 +210,58 @@ failed:
return key;
}
-TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *GUID_val)
+/* The caller is to provide a correctly sized key */
+int ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_val *GUID_val,
+ TDB_DATA *key)
{
- TDB_DATA key;
- const char *GUID_prefix = "GUID=";
- const int GUID_prefix_len = strlen(GUID_prefix);
-
- key.dptr = talloc_size(mem_ctx,
- GUID_val->length+GUID_prefix_len);
+ const char *GUID_prefix = LTDB_GUID_KEY_PREFIX;
+ const int GUID_prefix_len = sizeof(LTDB_GUID_KEY_PREFIX) - 1;
- if (key.dptr == NULL) {
- errno = ENOMEM;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ if (key->dsize != (GUID_val->length+GUID_prefix_len)) {
+ return LDB_ERR_OPERATIONS_ERROR;
}
- memcpy(key.dptr, "GUID=", GUID_prefix_len);
- memcpy(&key.dptr[GUID_prefix_len],
- GUID_val->data, GUID_val->length);
- key.dsize = talloc_get_size(key.dptr);
- return key;
+ memcpy(key->dptr, GUID_prefix, GUID_prefix_len);
+ memcpy(&key->dptr[GUID_prefix_len],
+ GUID_val->data, GUID_val->length);
+ return LDB_SUCCESS;
}
-TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *idx_val)
+/*
+ * The caller is to provide a correctly sized key, used only in
+ * the GUID index mode
+ */
+int ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val,
+ TDB_DATA *key)
{
- TDB_DATA key;
struct ldb_context *ldb = ldb_module_get_ctx(module);
struct ldb_dn *dn;
if (ltdb->cache->GUID_index_attribute != NULL) {
- return ltdb_guid_to_key(module, ltdb, mem_ctx, idx_val);
+ return ltdb_guid_to_key(module, ltdb,
+ idx_val, key);
}
dn = ldb_dn_from_ldb_val(mem_ctx, ldb, idx_val);
if (dn == NULL) {
- errno = EINVAL;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ /*
+ * LDB_ERR_INVALID_DN_SYNTAX would just be confusing
+ * to the caller, as this in an invalid index value
+ */
+ return LDB_ERR_OPERATIONS_ERROR;
}
/* form the key */
- key = ltdb_key_dn(module, mem_ctx, dn);
+ *key = ltdb_key_dn(module, mem_ctx, dn);
TALLOC_FREE(dn);
- if (!key.dptr) {
- errno = ENOMEM;
- key.dptr = NULL;
- key.dsize = 0;
- return key;
+ if (!key->dptr) {
+ return ldb_module_oom(module);
}
- return key;
+ return LDB_SUCCESS;
}
/*
@@ -283,6 +279,7 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
TDB_DATA key;
const struct ldb_val *guid_val;
+ int ret;
if (ltdb->cache->GUID_index_attribute == NULL) {
return ltdb_key_dn(module, mem_ctx, msg->dn);
@@ -301,8 +298,25 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
return key;
}
- return ltdb_guid_to_key(module, ltdb, mem_ctx, guid_val);
+ /* In this case, allocate with talloc */
+ key.dptr = talloc_size(mem_ctx, LTDB_GUID_KEY_SIZE);
+ if (key.dptr == NULL) {
+ errno = ENOMEM;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ key.dsize = talloc_get_size(key.dptr);
+ ret = ltdb_guid_to_key(module, ltdb, guid_val, &key);
+
+ if (ret != LDB_SUCCESS) {
+ errno = EINVAL;
+ key.dptr = NULL;
+ key.dsize = 0;
+ return key;
+ }
+ return key;
}
/*
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 0e6e9797d4b..cd77e3526ec 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -80,6 +80,7 @@ struct ltdb_context {
/* DB keys */
#define LTDB_GUID_KEY_PREFIX "GUID="
#define LTDB_GUID_SIZE 16
+#define LTDB_GUID_KEY_SIZE (LTDB_GUID_SIZE + sizeof(LTDB_GUID_KEY_PREFIX) - 1)
/* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c */
@@ -151,14 +152,15 @@ TDB_DATA ltdb_key_dn(struct ldb_module *module, TALLOC_CTX *mem_ctx,
struct ldb_dn *dn);
TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
const struct ldb_message *msg);
-TDB_DATA ltdb_guid_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *guid_val);
-TDB_DATA ltdb_idx_to_key(struct ldb_module *module,
- struct ltdb_private *ltdb,
- TALLOC_CTX *mem_ctx,
- const struct ldb_val *idx_val);
+int ltdb_guid_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const struct ldb_val *GUID_val,
+ TDB_DATA *key);
+int ltdb_idx_to_key(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ TALLOC_CTX *mem_ctx,
+ const struct ldb_val *idx_val,
+ TDB_DATA *key);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req);
int ltdb_delete_noindex(struct ldb_module *module,
--
2.11.0
From ae9df7afef307e19f28e29f3381810d855a89c34 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 22:25:43 +1200
Subject: [PATCH 67/88] ldb_tdb: Clean up list in after use in
ltdb_key_dn_from_idx()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 78675e8ae56..765198bfd80 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -464,10 +464,12 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
ret = ltdb_index_dn_base_dn(module, ltdb, dn, list);
if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(list);
return ret;
}
if (list->count == 0) {
+ TALLOC_FREE(list);
return LDB_ERR_NO_SUCH_OBJECT;
}
if (list->count > 1) {
@@ -486,6 +488,7 @@ int ltdb_key_dn_from_idx(struct ldb_module *module,
/* The tdb_key memory is allocated by the caller */
ret = ltdb_guid_to_key(module, ltdb,
&list->dn[0], tdb_key);
+ TALLOC_FREE(list);
if (ret != LDB_SUCCESS) {
return LDB_ERR_OPERATIONS_ERROR;
--
2.11.0
From dac67d4d884c5ad38a33fb117fd3af1d63fb491f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 1 Sep 2017 20:06:50 +1200
Subject: [PATCH 68/88] ldb_tdb: Avoid canonicalise and base64 work for DN
values, these are already OK
This is important with the GUID index, as a DN lookup is much more common now.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 103 ++++++++++++++++++++++++++++++++------------
1 file changed, 76 insertions(+), 27 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 765198bfd80..48d80da4654 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -733,45 +733,89 @@ int ltdb_index_transaction_cancel(struct ldb_module *module)
the caller is responsible for freeing
*/
static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
+ struct ltdb_private *ltdb,
const char *attr, const struct ldb_val *value,
const struct ldb_schema_attribute **ap)
{
struct ldb_dn *ret;
struct ldb_val v;
- const struct ldb_schema_attribute *a;
- char *attr_folded;
+ const struct ldb_schema_attribute *a = NULL;
+ char *attr_folded = NULL;
+ const char *attr_for_dn = NULL;
int r;
+ bool should_b64_encode;
- attr_folded = ldb_attr_casefold(ldb, attr);
- if (!attr_folded) {
- return NULL;
+ if (attr[0] == '@') {
+ attr_for_dn = attr;
+ v = *value;
+ if (ap != NULL) {
+ *ap = NULL;
+ }
+ } else {
+ attr_folded = ldb_attr_casefold(ldb, attr);
+ if (!attr_folded) {
+ return NULL;
+ }
+
+ attr_for_dn = attr_folded;
+
+ a = ldb_schema_attribute_by_name(ldb, attr);
+ if (ap) {
+ *ap = a;
+ }
+ r = a->syntax->canonicalise_fn(ldb, ldb, value, &v);
+ if (r != LDB_SUCCESS) {
+ const char *errstr = ldb_errstring(ldb);
+ /* canonicalisation can be refused. For
+ example, a attribute that takes wildcards
+ will refuse to canonicalise if the value
+ contains a wildcard */
+ ldb_asprintf_errstring(ldb,
+ "Failed to create index "
+ "key for attribute '%s':%s%s%s",
+ attr, ldb_strerror(r),
+ (errstr?":":""),
+ (errstr?errstr:""));
+ talloc_free(attr_folded);
+ return NULL;
+ }
}
- a = ldb_schema_attribute_by_name(ldb, attr);
- if (ap) {
- *ap = a;
- }
- r = a->syntax->canonicalise_fn(ldb, ldb, value, &v);
- if (r != LDB_SUCCESS) {
- const char *errstr = ldb_errstring(ldb);
- /* canonicalisation can be refused. For example,
- a attribute that takes wildcards will refuse to canonicalise
- if the value contains a wildcard */
- ldb_asprintf_errstring(ldb, "Failed to create index key for attribute '%s':%s%s%s",
- attr, ldb_strerror(r), (errstr?":":""), (errstr?errstr:""));
- talloc_free(attr_folded);
- return NULL;
+ /*
+ * We do not base 64 encode a DN in a key, it has already been
+ * casefold and lineraized, that is good enough. That already
+ * avoids embedded NUL etc.
+ */
+ if (ltdb->cache->GUID_index_attribute != NULL) {
+ if (strcmp(attr, LTDB_IDXDN) == 0) {
+ should_b64_encode = false;
+ } else if (strcmp(attr, LTDB_IDXONE) == 0) {
+ /*
+ * We can only change the behaviour for IDXONE
+ * when the GUID index is enabled
+ */
+ should_b64_encode = false;
+ } else {
+ should_b64_encode
+ = ldb_should_b64_encode(ldb, &v);
+ }
+ } else {
+ should_b64_encode = ldb_should_b64_encode(ldb, &v);
}
- if (ldb_should_b64_encode(ldb, &v)) {
+
+ if (should_b64_encode) {
char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
if (!vstr) {
talloc_free(attr_folded);
return NULL;
}
- ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s::%s", LTDB_INDEX, attr_folded, vstr);
+ ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s::%s", LTDB_INDEX,
+ attr_for_dn, vstr);
talloc_free(vstr);
} else {
- ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s:%.*s", LTDB_INDEX, attr_folded, (int)v.length, (char *)v.data);
+ ret = ldb_dn_new_fmt(ldb, ldb, "%s:%s:%.*s", LTDB_INDEX,
+ attr_for_dn,
+ (int)v.length, (char *)v.data);
}
if (v.data != value->data) {
@@ -870,7 +914,9 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
/* the attribute is indexed. Pull the list of DNs that match the
search criterion */
- dn = ltdb_index_key(ldb, tree->u.equality.attr, &tree->u.equality.value, NULL);
+ dn = ltdb_index_key(ldb, ltdb,
+ tree->u.equality.attr,
+ &tree->u.equality.value, NULL);
if (!dn) return LDB_ERR_OPERATIONS_ERROR;
ret = ltdb_dn_list_load(module, ltdb, dn, list);
@@ -1314,7 +1360,7 @@ static int ltdb_index_dn_attr(struct ldb_module *module,
/* work out the index key from the parent DN */
val.data = (uint8_t *)((uintptr_t)ldb_dn_get_casefold(dn));
val.length = strlen((char *)val.data);
- key = ltdb_index_key(ldb, attr, &val, NULL);
+ key = ltdb_index_key(ldb, ltdb, attr, &val, NULL);
if (!key) {
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1671,7 +1717,8 @@ static int ltdb_index_add1(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- dn_key = ltdb_index_key(ldb, el->name, &el->values[v_idx], &a);
+ dn_key = ltdb_index_key(ldb, ltdb,
+ el->name, &el->values[v_idx], &a);
if (!dn_key) {
talloc_free(list);
return LDB_ERR_OPERATIONS_ERROR;
@@ -1689,7 +1736,8 @@ static int ltdb_index_add1(struct ldb_module *module,
* DN -> GUID record
*/
if (list->count > 0 &&
- (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX ||
+ ((a != NULL
+ && (a->flags & LDB_ATTR_FLAG_UNIQUE_INDEX)) ||
ldb_attr_cmp(el->name, LTDB_IDXDN) == 0)) {
/*
* We do not want to print info about a possibly
@@ -2020,7 +2068,8 @@ int ltdb_index_del_value(struct ldb_module *module,
return LDB_SUCCESS;
}
- dn_key = ltdb_index_key(ldb, el->name, &el->values[v_idx], NULL);
+ dn_key = ltdb_index_key(ldb, ltdb,
+ el->name, &el->values[v_idx], NULL);
if (!dn_key) {
return LDB_ERR_OPERATIONS_ERROR;
}
--
2.11.0
From 5dcc742535f393d6cec706e222c1f7cae71dddfb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:19:39 +1200
Subject: [PATCH 69/88] ldb_tdb: Give a good error message on add without an
objectGUID
(or whatever the @IDX_GUID value is)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index b4ce2470352..82ebe735729 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -292,6 +292,12 @@ TDB_DATA ltdb_key_msg(struct ldb_module *module, TALLOC_CTX *mem_ctx,
guid_val = ldb_msg_find_ldb_val(msg,
ltdb->cache->GUID_index_attribute);
if (guid_val == NULL) {
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Did not find GUID attribute %s "
+ "in %s, required for TDB record "
+ "key in " LTDB_IDXGUID " mode.",
+ ltdb->cache->GUID_index_attribute,
+ ldb_dn_get_linearized(msg->dn));
errno = EINVAL;
key.dptr = NULL;
key.dsize = 0;
--
2.11.0
From 73251d0606074037d591081790e50428b35b4e4a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 15:30:08 +1200
Subject: [PATCH 70/88] ldb_tdb: Describe index format and control points
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 113 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 48d80da4654..aee481f92b3 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -149,6 +149,119 @@
* Author: Andrew Tridgell
*/
+/*
+
+LDB Index design and choice of TDB key:
+=======================================
+
+LDB has index records held as LDB objects with a special record like:
+
+dn: @INDEX:attr:value
+
+value may be base64 encoded, if it is deemed not printable:
+
+dn: @INDEX:attr::base64-value
+
+In each record, there is two possible formats:
+
+The original format is:
+-----------------------
+
+dn: @INDEX:NAME:DNSUPDATEPROXY
+ at IDXVERSION: 2
+ at IDX: CN=DnsUpdateProxy,CN=Users,DC=addom,DC=samba,DC=example,DC=com
+
+In this format, @IDX is multi-valued, one entry for each match
+
+The corrosponding entry is stored in a TDB record with key:
+
+DN=CN=DNSUPDATEPROXY,CN=USERS,DC=ADDOM,DC=SAMBA,DC=EXAMPLE,DC=COM
+
+(This allows a scope BASE search to directly find the record via
+a simple casefold of the DN).
+
+The original mixed-case DN is stored in the entry iself.
+
+
+The new 'GUID index' format is:
+-------------------------------
+
+dn: @INDEX:NAME:DNSUPDATEPROXY
+ at IDXVERSION: 3
+ at IDX: <binary GUID>[<binary GUID>[...]]
+
+The binary guid is 16 bytes, as bytes and not expanded as hexidecimal
+or pretty-printed. The GUID is chosen from the message to be stored
+by the @IDXGUID attribute on @INDEXLIST.
+
+If there are multiple values the @IDX value simply becomes longer,
+in multiples of 16.
+
+The corrosponding entry is stored in a TDB record with key:
+
+GUID=<binary GUID>
+
+This allows a very quick translation between the fixed-length index
+values and the TDB key, while seperating entries from other data
+in the TDB, should they be unlucky enough to start with the bytes of
+the 'DN=' prefix.
+
+Additionally, this allows a scope BASE search to directly find the
+record via a simple match on a GUID= extended DN, controlled via
+ at IDX_DN_GUID on @INDEXLIST
+
+Exception for special @ DNs:
+
+ at BASEINFO, @INDEXLIST and all other special DNs are stored as per the
+original format, as they are never referenced in an index and are used
+to bootstrap the database.
+
+
+Control points for choice of index mode
+---------------------------------------
+
+The choice of index and TDB key mode is made based (for example, from
+Samba) on entries in the @INDEXLIST DN:
+
+dn: @INDEXLIST
+ at IDXGUID: objectGUID
+ at IDX_DN_GUID: GUID
+
+By default, the original DN format is used.
+
+
+Control points for choosing indexed attributes
+----------------------------------------------
+
+ at IDXATTR controls if an attribute is indexed
+
+dn: @INDEXLIST
+ at IDXATTR: samAccountName
+ at IDXATTR: nETBIOSName
+
+
+C Override functions
+--------------------
+
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+ const char *GUID_index_attribute,
+ const char *GUID_index_dn_component)
+
+This is used, particularly in combination with the below, instead of
+the @IDXGUID and @IDX_DN_GUID values in @INDEXLIST.
+
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+ bool one_level_indexes);
+void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
+ ldb_attribute_handler_override_fn_t override,
+ void *private_data);
+
+When the above two functions are called in combination, the @INDEXLIST
+values are not read from the DB, so
+ldb_schema_set_override_GUID_index() must be called.
+
+*/
+
#include "ldb_tdb.h"
#include "ldb_private.h"
--
2.11.0
From 81c0e1dda412e9cda15fbe27dfc4478f6b1e0600 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 18:07:51 +1200
Subject: [PATCH 71/88] ldb_tdb: Clean up index records on ltdb_index_add_new()
failure.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index aee481f92b3..780bfedef65 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2147,10 +2147,29 @@ int ltdb_index_add_new(struct ldb_module *module,
ret = ltdb_index_add_all(module, ltdb, msg);
if (ret != LDB_SUCCESS) {
+ /*
+ * Because we can't trust the caller to be doing
+ * transactions properly, clean up any index for this
+ * entry rather than relying on a transaction
+ * cleanup
+ */
+
+ ltdb_index_delete(module, msg);
return ret;
}
- return ltdb_index_onelevel(module, msg, 1);
+ ret = ltdb_index_onelevel(module, msg, 1);
+ if (ret != LDB_SUCCESS) {
+ /*
+ * Because we can't trust the caller to be doing
+ * transactions properly, clean up any index for this
+ * entry rather than relying on a transaction
+ * cleanup
+ */
+ ltdb_index_delete(module, msg);
+ return ret;
+ }
+ return ret;
}
--
2.11.0
From 5dd01c57eee9e5caed24a93e9ba1602600da9305 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 10:02:16 +1200
Subject: [PATCH 72/88] ldb_tdb: Remove LTDB_FLAG_HIDDEN and ignore "HIDDEN" in
@ATTRIBUTES
This was (unintentionally) disabled by
6ef61825541131e16a03975cdbd344e2bbebf810 in 2006.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_cache.c | 25 +++++++++++--------------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 7bf5f156b45..5347a3be13b 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -36,7 +36,6 @@
#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
#define LTDB_FLAG_INTEGER (1<<1)
-#define LTDB_FLAG_HIDDEN (1<<2)
/* valid attribute flags */
static const struct {
@@ -45,7 +44,7 @@ static const struct {
} ltdb_valid_attr_flags[] = {
{ "CASE_INSENSITIVE", LTDB_FLAG_CASE_INSENSITIVE },
{ "INTEGER", LTDB_FLAG_INTEGER },
- { "HIDDEN", LTDB_FLAG_HIDDEN },
+ { "HIDDEN", 0 },
{ "NONE", 0 },
{ NULL, 0 }
};
@@ -150,7 +149,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
/* mapping these flags onto ldap 'syntaxes' isn't strictly correct,
but its close enough for now */
for (i=0;i<attrs_msg->num_elements;i++) {
- unsigned flags;
+ unsigned flags, attr_flags;
const char *syntax;
const struct ldb_schema_syntax *s;
const struct ldb_schema_attribute *a =
@@ -167,17 +166,15 @@ static int ltdb_attributes_load(struct ldb_module *module)
attrs_msg->elements[i].name);
goto failed;
}
- switch (flags & ~LTDB_FLAG_HIDDEN) {
- case 0:
- syntax = LDB_SYNTAX_OCTET_STRING;
- break;
- case LTDB_FLAG_CASE_INSENSITIVE:
+
+ /* These are not currently flags, each is exclusive */
+ if (flags == LTDB_FLAG_CASE_INSENSITIVE) {
syntax = LDB_SYNTAX_DIRECTORY_STRING;
- break;
- case LTDB_FLAG_INTEGER:
+ } else if (flags == LTDB_FLAG_INTEGER) {
syntax = LDB_SYNTAX_INTEGER;
- break;
- default:
+ } else if (flags == 0) {
+ syntax = LDB_SYNTAX_OCTET_STRING;
+ } else {
ldb_debug(ldb, LDB_DEBUG_ERROR,
"Invalid flag combination 0x%x for '%s' "
"in @ATTRIBUTES",
@@ -194,12 +191,12 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
- flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
+ attr_flags = LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
r = ldb_schema_attribute_fill_with_syntax(ldb,
attrs,
attrs_msg->elements[i].name,
- flags, s,
+ attr_flags, s,
&attrs[num_loaded_attrs + ldb->schema.num_attributes]);
if (r != 0) {
goto failed;
--
2.11.0
From 397539a4bf5c750454865c21e1e7bdf88a14c580 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 12:36:43 +1200
Subject: [PATCH 73/88] ldb_tdb: Add UNIQUE_INDEX as a possible flag
This allows easy testing of our unique index code and behaivour from python
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_cache.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 5347a3be13b..5b90bd99f47 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -36,6 +36,7 @@
#define LTDB_FLAG_CASE_INSENSITIVE (1<<0)
#define LTDB_FLAG_INTEGER (1<<1)
+#define LTDB_FLAG_UNIQUE_INDEX (1<<2)
/* valid attribute flags */
static const struct {
@@ -45,6 +46,7 @@ static const struct {
{ "CASE_INSENSITIVE", LTDB_FLAG_CASE_INSENSITIVE },
{ "INTEGER", LTDB_FLAG_INTEGER },
{ "HIDDEN", 0 },
+ { "UNIQUE_INDEX", LTDB_FLAG_UNIQUE_INDEX},
{ "NONE", 0 },
{ NULL, 0 }
};
@@ -149,7 +151,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
/* mapping these flags onto ldap 'syntaxes' isn't strictly correct,
but its close enough for now */
for (i=0;i<attrs_msg->num_elements;i++) {
- unsigned flags, attr_flags;
+ unsigned flags = 0, attr_flags = 0;
const char *syntax;
const struct ldb_schema_syntax *s;
const struct ldb_schema_attribute *a =
@@ -167,6 +169,11 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
+ if (flags & LTDB_FLAG_UNIQUE_INDEX) {
+ attr_flags = LDB_ATTR_FLAG_UNIQUE_INDEX;
+ }
+ flags &= ~LTDB_FLAG_UNIQUE_INDEX;
+
/* These are not currently flags, each is exclusive */
if (flags == LTDB_FLAG_CASE_INSENSITIVE) {
syntax = LDB_SYNTAX_DIRECTORY_STRING;
@@ -191,7 +198,7 @@ static int ltdb_attributes_load(struct ldb_module *module)
goto failed;
}
- attr_flags = LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
+ attr_flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB;
r = ldb_schema_attribute_fill_with_syntax(ldb,
attrs,
--
2.11.0
From f0663d33f73e55fad93926e4a790378f0f2407fd Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 11:28:23 +1200
Subject: [PATCH 74/88] ldb: Add more tests covering in-transaction GUID index
and unique index behaviour
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++++++++++++++---
1 file changed, 173 insertions(+), 12 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 4c35e44ee28..b2e31c23f4e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -984,6 +984,8 @@ class AddModifyTests(TestCase):
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
+ self.l.add({"dn": "@ATTRIBUTES",
+ "objectUUID": "UNIQUE_INDEX"})
def test_add_dup(self):
self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
@@ -1023,6 +1025,88 @@ class AddModifyTests(TestCase):
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde2"})
+ def test_add_move_fail_move_move(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on duplicate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ self.l.rename("OU=DUP2,DC=SAMBA,DC=ORG",
+ "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+
+ res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde1)")
+ self.assertEqual(len(res2), 1)
+ self.assertEqual(str(res2[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
+ res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde2)")
+ self.assertEqual(len(res3), 1)
+ self.assertEqual(str(res3[0].dn), "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ def test_move_missing(self):
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on missing")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_NO_SUCH_OBJECT)
+
+ def test_move_missing2(self):
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on missing")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_NO_SUCH_OBJECT)
+
+ def test_move_fail_move_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+ try:
+ self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
+ "OU=DUP2,DC=SAMBA,DC=ORG")
+ self.fail("Should have failed on duplicate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ self.l.rename("OU=DUP2,DC=SAMBA,DC=ORG",
+ "OU=DUP3,DC=SAMBA,DC=ORG")
+
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde3"})
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
@@ -1030,20 +1114,9 @@ class IndexedAddModifyTests(AddModifyTests):
def setUp(self):
super(IndexedAddModifyTests, self).setUp()
self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
"@IDXONE": [b"1"]})
-class GUIDIndexedAddModifyTests(AddModifyTests):
- """Test searches using the index, to ensure the index doesn't
- break things"""
- def setUp(self):
- super(GUIDIndexedAddModifyTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
-
def test_duplicate_GUID(self):
try:
self.l.add({"dn": "OU=DUPGUID,DC=SAMBA,DC=ORG",
@@ -1055,6 +1128,94 @@ class GUIDIndexedAddModifyTests(AddModifyTests):
enum = err.args[0]
self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+ def test_duplicate_name_dup_GUID(self):
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"a123456789abcdef"})
+ try:
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"a123456789abcdef"})
+ self.fail("Should have failed adding dupliate GUID")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ def test_duplicate_name_dup_GUID2(self):
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"abc3456789abcdef"})
+ try:
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"aaa3456789abcdef"})
+ self.fail("Should have failed adding dupliate DN")
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_ENTRY_ALREADY_EXISTS)
+
+ # Checking the GUID didn't stick in the index
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"aaa3456789abcdef"})
+
+ def test_add_dup_guid_add(self):
+ self.l.add({"dn": "OU=DUP,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ try:
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde1"})
+ self.fail("Should have failed on duplicate GUID")
+
+ except ldb.LdbError as err:
+ enum = err.args[0]
+ self.assertEqual(enum, ldb.ERR_CONSTRAINT_VIOLATION)
+
+ self.l.add({"dn": "OU=DUP2,DC=SAMBA,DC=ORG",
+ "name": b"Admins",
+ "x": "z", "y": "a",
+ "objectUUID": b"0123456789abcde2"})
+
+class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
+ """Test searches using the index, to ensure the index doesn't
+ break things"""
+ def setUp(self):
+ super(GUIDIndexedAddModifyTests, self).setUp()
+ indexlist = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
+ m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
+ self.l.modify(m)
+
+
+class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
+ """Test GUID index behaviour insdie the transaction"""
+ def setUp(self):
+ super(GUIDTransIndexedAddModifyTests, self).setUp()
+ self.l.transaction_start()
+
+ def tearDown(self):
+ self.l.transaction_commit()
+
+class TransIndexedAddModifyTests(IndexedAddModifyTests):
+ """Test index behaviour insdie the transaction"""
+ def setUp(self):
+ super(TransIndexedAddModifyTests, self).setUp()
+ self.l.transaction_start()
+
+ def tearDown(self):
+ self.l.transaction_commit()
class DnTests(TestCase):
--
2.11.0
From 867bdf622750f72dc3178802586c1dff19b4a02f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 11:49:02 +1200
Subject: [PATCH 75/88] ldb_tdb: Remove incorrect early return from re-index
The ltdb->cache->attribute_indexes test is not correct with the GUID index mode
so for consistency remove it. This will make re-index on a large un-indexed
database slower, but that is better than making the wrong choice on a large
GUID-indexed database.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 5 -----
1 file changed, 5 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 780bfedef65..dc39b6f27e6 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2597,11 +2597,6 @@ int ltdb_reindex(struct ldb_module *module)
return LDB_ERR_OPERATIONS_ERROR;
}
- /* if we don't have indexes we have nothing todo */
- if (!ltdb->cache->attribute_indexes) {
- return LDB_SUCCESS;
- }
-
ctx.module = module;
ctx.error = 0;
--
2.11.0
From d2243cd8bea78fabc8f23c74eac5276231f6acf7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 13:16:31 +1200
Subject: [PATCH 76/88] ldb_tdb: Print progress messages on re-index
A re-index of 10,000 entries is slow enough and rare enought that we can
justify the message being at LDB_DEBUG_WARNING as otherwise the administrator
will be sure the "lockup" was one.
The default for ldb is to print LDB_DEBUG_WARNING in comand-line tools
and the default for Samba is to log it at level 2.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index dc39b6f27e6..65127d1de2a 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2373,6 +2373,7 @@ static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, vo
struct ltdb_reindex_context {
struct ldb_module *module;
int error;
+ uint32_t count;
};
/*
@@ -2477,6 +2478,13 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st
talloc_free(msg);
+ ctx->count++;
+ if (ctx->count % 10000 == 0) {
+ ldb_debug(ldb, LDB_DEBUG_WARNING,
+ "Reindexing: re-keyed %u records so far",
+ ctx->count);
+ }
+
return 0;
}
@@ -2558,6 +2566,13 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *
talloc_free(msg);
+ ctx->count++;
+ if (ctx->count % 10000 == 0) {
+ ldb_debug(ldb, LDB_DEBUG_WARNING,
+ "Reindexing: re-indexed %u records so far",
+ ctx->count);
+ }
+
return 0;
}
@@ -2599,6 +2614,7 @@ int ltdb_reindex(struct ldb_module *module)
ctx.module = module;
ctx.error = 0;
+ ctx.count = 0;
/* now traverse adding any indexes for normal LDB records */
ret = tdb_traverse(ltdb->tdb, re_key, &ctx);
@@ -2616,6 +2632,7 @@ int ltdb_reindex(struct ldb_module *module)
}
ctx.error = 0;
+ ctx.count = 0;
/* now traverse adding any indexes for normal LDB records */
ret = tdb_traverse(ltdb->tdb, re_index, &ctx);
@@ -2632,5 +2649,11 @@ int ltdb_reindex(struct ldb_module *module)
return ctx.error;
}
+ if (ctx.count > 10000) {
+ ldb_debug(ldb_module_get_ctx(module),
+ LDB_DEBUG_WARNING, "Reindexing: re_index successful on %s, "
+ "final index write-out will be in transaction commit",
+ tdb_name(ltdb->tdb));
+ }
return LDB_SUCCESS;
}
--
2.11.0
From 9d86e87af4648c12a8147ca9e4d57e04a613fff5 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 22:00:40 +1200
Subject: [PATCH 77/88] ldb_tdb: Update comments for the delete_index() pass of
ltdb_reindex()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 65127d1de2a..aea869a0b86 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2333,7 +2333,12 @@ int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg)
/*
- traversal function that deletes all @INDEX records
+ traversal function that deletes all @INDEX records in the in-memory
+ TDB.
+
+ This does not touch the actual DB, that is done at transaction
+ commit, which in turn greatly reduces DB churn as we will likely
+ be able to do a direct update into the old record.
*/
static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state)
{
@@ -2358,6 +2363,11 @@ static int delete_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, vo
v.length = strnlen((char *)key.dptr, key.dsize) - 3;
dn = ldb_dn_from_ldb_val(ltdb, ldb_module_get_ctx(module), &v);
+
+ /*
+ * This does not actually touch the DB quite yet, just
+ * the in-memory index cache
+ */
ret = ltdb_dn_list_store(module, dn, &list);
if (ret != LDB_SUCCESS) {
ldb_asprintf_errstring(ldb_module_get_ctx(module),
--
2.11.0
From 28ce6bdc3509a832288c2b32dd5016e7d12f60d8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 13:00:08 +1200
Subject: [PATCH 78/88] ldb_tdb: Also ban a (indexed) search against like
@IDXDN=foo
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_index.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index aea869a0b86..d5c7e039375 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -1059,6 +1059,12 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
list->count = 0;
return LDB_SUCCESS;
}
+ if (tree->u.equality.attr[0] == '@') {
+ /* Do not allow a indexed search against an @ */
+ list->dn = NULL;
+ list->count = 0;
+ return LDB_SUCCESS;
+ }
if (ldb_attr_dn(tree->u.equality.attr) == 0) {
if (ltdb->cache->GUID_index_attribute == NULL) {
list->dn = talloc_array(list, struct ldb_val, 1);
--
2.11.0
From 41a41da28f6e27796e6761a136072fdeed2e94cb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:54:00 +1200
Subject: [PATCH 79/88] TODO: Release ldb 1.3.0
* GUID Index support.
NOTE: When activated by setting @IDXGUID in the @INDEXLIST dn, all
entries in the DB are re-keyed in a way that is NOT visible to
ldb 1.2.2 and earlier. To re-key back to the previous format, remove
the @IDXGUID attribute from @INDEXLIST using ldb 1.2.2 or later.
(ldb 1.2.2 can re-key, but not otherwise read, the new DB format).
* Give LDB_ERR_CONSTRAINT_VIOLATION, not LDB_ERR_ENTRY_ALREADY_EXISTS
when a duplicate value is detected in a unique index
* Print status information during a > 10,000 entry re-index
(as this can be slow)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ABI/ldb-1.3.0.sigs | 279 ++++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.3.0.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 284 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.3.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.3.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
diff --git a/lib/ldb/ABI/ldb-1.3.0.sigs b/lib/ldb/ABI/ldb-1.3.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.3.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.3.0.sigs b/lib/ldb/ABI/pyldb-util-1.3.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.3.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index bd17b7b9edc..de7eef72c14 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.2.2'
+VERSION = '1.3.0'
blddir = 'bin'
--
2.11.0
From e2db87d9507683d828f16c3befdedaca8738a414 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 25 Aug 2017 17:37:05 +1200
Subject: [PATCH 80/88] selftest: sort dbcheck output to avoid sort order
impacting results
The GUID index code will change the returned results order
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
testprogs/blackbox/dbcheck-links.sh | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh
index fb66d14e6f1..17973e97429 100755
--- a/testprogs/blackbox/dbcheck-links.sh
+++ b/testprogs/blackbox/dbcheck-links.sh
@@ -59,7 +59,9 @@ dbcheck() {
if [ "$?" != "1" ]; then
return 1
fi
- diff $tmpfile $release_dir/expected-dbcheck-link-output.txt
+ sort $tmpfile > $tmpfile.sorted
+ sort $release_dir/expected-dbcheck-link-output.txt > $tmpfile.expected
+ diff -u $tmpfile.sorted $tmpfile.expected
if [ "$?" != "0" ]; then
return 1
fi
--
2.11.0
From 021b4452d0c0fd999c94f6600e70266cb0e00467 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:21:29 +1200
Subject: [PATCH 81/88] provision: make clear that the tmp ldb is running in
@IDXGUID mode
This happended when the schema was set on the DB, forcing the full set of Samba behaviours
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
python/samba/schema.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/python/samba/schema.py b/python/samba/schema.py
index 839d681825a..17a171960b9 100644
--- a/python/samba/schema.py
+++ b/python/samba/schema.py
@@ -126,12 +126,15 @@ class Schema(object):
self.ldb.connect(url=schemadb_path)
self.ldb.transaction_start()
try:
+ # These are actually ignored, as the schema has been forced
+ # when the ldb object was created, and that overrides this
self.ldb.add_ldif("""dn: @ATTRIBUTES
linkID: INTEGER
dn: @INDEXLIST
@IDXATTR: linkID
@IDXATTR: attributeSyntax
+ at IDXGUID: objectGUID
""")
# These bits of LDIF are supplied when the Schema object is created
self.ldb.add_ldif(self.schema_dn_add)
--
2.11.0
From 2d7fdf6ee3f894249f3318c6b6bd717421c5f069 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:31:21 +1200
Subject: [PATCH 82/88] provision: Add a fixed objectGUID to the tmp DB used
for LDAP backend schema work
This DB holds a copy of the schema, but now needs to have an objectGUID on each record.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
python/samba/schema.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/python/samba/schema.py b/python/samba/schema.py
index 17a171960b9..3828003f4cf 100644
--- a/python/samba/schema.py
+++ b/python/samba/schema.py
@@ -136,8 +136,12 @@ dn: @INDEXLIST
@IDXATTR: attributeSyntax
@IDXGUID: objectGUID
""")
+
+ schema_dn_add = self.schema_dn_add \
+ + "objectGUID: 24e2ca70-b093-4ae8-84c0-2d7ac652a1b8\n"
+
# These bits of LDIF are supplied when the Schema object is created
- self.ldb.add_ldif(self.schema_dn_add)
+ self.ldb.add_ldif(schema_dn_add)
self.ldb.modify_ldif(self.schema_dn_modify)
self.ldb.add_ldif(self.schema_data)
except:
--
2.11.0
From 6c81048cc8625f99dc816e4bed7ac1f7b36f428b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Sat, 2 Sep 2017 16:33:19 +1200
Subject: [PATCH 83/88] provision: Add a fixed GUID to the samba4top
objectclass definition
This is only used in the OpenLDAP backend and will certainly be removed before this becomes production.
(a production backend will use the real AD top objectclass)
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/setup/schema_samba4.ldif | 1 +
1 file changed, 1 insertion(+)
diff --git a/source4/setup/schema_samba4.ldif b/source4/setup/schema_samba4.ldif
index fcfaf9819b7..4fd729e3149 100644
--- a/source4/setup/schema_samba4.ldif
+++ b/source4/setup/schema_samba4.ldif
@@ -293,6 +293,7 @@ objectClass: top
objectClass: classSchema
cn: Samba4Top
subClassOf: top
+objectGUID: 4af54ff0-ff3c-4f17-8fb0-611ec83ddfb4
governsID: 1.3.6.1.4.1.7165.4.2.1
mayContain: msDS-ObjectReferenceBL
rDNAttID: cn
--
2.11.0
From a11a5724920fbacb6d9ffa193148e0fe5a105ffb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 15 Aug 2017 15:58:57 +1200
Subject: [PATCH 84/88] dsdb: Set that Samba uses the GUID index in LDB
This is optional, but only to aid the downgrade script (and in case
there is some major issue found with it). We don't support that mode,
as that would require us to test and maintain multiple code paths and
not optimise queries to be GUID centric.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/dsdb/schema/schema_set.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 8141e323e50..23d881f0bdb 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -69,9 +69,16 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
struct ldb_message *msg;
struct ldb_message *msg_idx;
+ struct loadparm_context *lp_ctx =
+ talloc_get_type(ldb_get_opaque(ldb, "loadparm"),
+ struct loadparm_context);
/* setup our own attribute name to schema handler */
ldb_schema_attribute_set_override_handler(ldb, dsdb_attribute_handler_override, schema);
ldb_schema_set_override_indexlist(ldb, true);
+ if (lp_ctx == NULL ||
+ lpcfg_parm_bool(lp_ctx, NULL, "dsdb", "guid index", true)) {
+ ldb_schema_set_override_GUID_index(ldb, "objectGUID", "GUID");
+ }
if (!write_indices_and_attributes) {
return ret;
@@ -108,6 +115,18 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
goto op_error;
}
+ if (lp_ctx == NULL ||
+ lpcfg_parm_bool(lp_ctx, NULL, "dsdb", "guid index", true)) {
+ ret = ldb_msg_add_string(msg_idx, "@IDXGUID", "objectGUID");
+ if (ret != LDB_SUCCESS) {
+ goto op_error;
+ }
+
+ ret = ldb_msg_add_string(msg_idx, "@IDX_DN_GUID", "GUID");
+ if (ret != LDB_SUCCESS) {
+ goto op_error;
+ }
+ }
ret = ldb_msg_add_string(msg_idx, "@IDXVERSION", SAMDB_INDEXING_VERSION);
if (ret != LDB_SUCCESS) {
--
2.11.0
From 6e7de66240d942437117feee5a5e33754bdb2976 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 8 Sep 2017 15:31:55 +1200
Subject: [PATCH 85/88] Do not re-use the attribute @IDXVERSION for
SAMDB_INDEXING_VERSION
Confusing these two concepts is not a good idea, SAMDB_INDEXING_VERSION refers to
a change in a Samba rule to canonicalise one of our attributes, not the
in-DB index format.
As we already change @INDEXLIST in this version, this commit
is at no extra cost.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/dsdb/schema/schema_set.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 23d881f0bdb..ca7a307212a 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -128,7 +128,7 @@ int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb,
}
}
- ret = ldb_msg_add_string(msg_idx, "@IDXVERSION", SAMDB_INDEXING_VERSION);
+ ret = ldb_msg_add_string(msg_idx, "@SAMDB_INDEXING_VERSION", SAMDB_INDEXING_VERSION);
if (ret != LDB_SUCCESS) {
goto op_error;
}
--
2.11.0
From 33fc146c0d6d247dc7c489eb77de7d216308d8a8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 13:53:19 +1200
Subject: [PATCH 86/88] dsdb: Only trigger a re-index once per @INDEXLIST
modification
A modify of both @INDEXLIST and @ATTRIBUTES will still trigger two re-index passes
but that is a task for later.
BUG: https://bugzilla.samba.org/show_bug.cgi?id=9527
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/dsdb/samdb/ldb_modules/partition.c | 90 +++++++++++++++++++++++++-----
1 file changed, 75 insertions(+), 15 deletions(-)
diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c
index c304efa645d..426fce36e4a 100644
--- a/source4/dsdb/samdb/ldb_modules/partition.c
+++ b/source4/dsdb/samdb/ldb_modules/partition.c
@@ -432,30 +432,90 @@ static int partition_copy_all(struct ldb_module *module,
return search_ret;
}
- /* now delete the object in the other partitions. Once that is
- done we will re-add the object, if search_ret was not
- LDB_ERR_NO_SUCH_OBJECT
+ /* now delete the object in the other partitions, if requried
*/
+ if (search_ret == LDB_ERR_NO_SUCH_OBJECT) {
+ for (i=0; data->partitions && data->partitions[i]; i++) {
+ int pret;
+ pret = dsdb_module_del(data->partitions[i]->module,
+ dn,
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+ if (pret != LDB_SUCCESS && pret != LDB_ERR_NO_SUCH_OBJECT) {
+ /* we should only get success or no
+ such object from the other partitions */
+ return pret;
+ }
+ }
+
+ return ldb_module_done(req, NULL, NULL, LDB_SUCCESS);
+ }
+
+ /* now add/modify in the other partitions */
for (i=0; data->partitions && data->partitions[i]; i++) {
+ struct ldb_message *modify_msg = NULL;
int pret;
- pret = dsdb_module_del(data->partitions[i]->module, dn, DSDB_FLAG_NEXT_MODULE, req);
- if (pret != LDB_SUCCESS && pret != LDB_ERR_NO_SUCH_OBJECT) {
- /* we should only get success or no
- such object from the other partitions */
+ unsigned int el_idx;
+
+ pret = dsdb_module_add(data->partitions[i]->module,
+ res->msgs[0],
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+ if (pret == LDB_SUCCESS) {
+ continue;
+ }
+
+ if (pret != LDB_ERR_ENTRY_ALREADY_EXISTS) {
return pret;
}
- }
+ modify_msg = ldb_msg_copy(req, res->msgs[0]);
+ if (modify_msg == NULL) {
+ return ldb_module_oom(module);
+ }
- if (search_ret != LDB_ERR_NO_SUCH_OBJECT) {
- /* now re-add in the other partitions */
- for (i=0; data->partitions && data->partitions[i]; i++) {
- int pret;
- pret = dsdb_module_add(data->partitions[i]->module, res->msgs[0], DSDB_FLAG_NEXT_MODULE, req);
- if (pret != LDB_SUCCESS) {
- return pret;
+ /*
+ * mark all the message elements as
+ * LDB_FLAG_MOD_REPLACE
+ */
+ for (el_idx=0;
+ el_idx < modify_msg->num_elements;
+ el_idx++) {
+ modify_msg->elements[el_idx].flags
+ = LDB_FLAG_MOD_REPLACE;
+ }
+
+ if (req->operation == LDB_MODIFY) {
+ const struct ldb_message *req_msg = req->op.mod.message;
+ /*
+ * mark elements to be removed, if there were
+ * deleted entirely above we need to delete
+ * them here too
+ */
+ for (el_idx=0; el_idx < req_msg->num_elements; el_idx++) {
+ if (req_msg->elements[el_idx].flags & LDB_FLAG_MOD_DELETE
+ || ((req_msg->elements[el_idx].flags & LDB_FLAG_MOD_REPLACE) &&
+ req_msg->elements[el_idx].num_values == 0)) {
+ if (ldb_msg_find_element(modify_msg,
+ req_msg->elements[el_idx].name) != NULL) {
+ continue;
+ }
+ ldb_msg_add_empty(modify_msg,
+ req_msg->elements[el_idx].name,
+ LDB_FLAG_MOD_REPLACE,
+ NULL);
+ }
}
}
+
+ pret = dsdb_module_modify(data->partitions[i]->module,
+ modify_msg,
+ DSDB_FLAG_NEXT_MODULE,
+ req);
+
+ if (pret != LDB_SUCCESS) {
+ return pret;
+ }
}
return ldb_module_done(req, NULL, NULL, LDB_SUCCESS);
--
2.11.0
From 47daab20506d491e743031994cef200985c6b7d2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 11 Sep 2017 21:39:44 +1200
Subject: [PATCH 87/88] scripting: Add script (backportable) to undo a GUID
index
This script allows the DB to be read, and re-indexed, by an earlier Samba version,
most likely 4.7 with some backported patches.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
source4/scripting/bin/sambaundoguididx | 84 ++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
create mode 100755 source4/scripting/bin/sambaundoguididx
diff --git a/source4/scripting/bin/sambaundoguididx b/source4/scripting/bin/sambaundoguididx
new file mode 100755
index 00000000000..24a95e20d7f
--- /dev/null
+++ b/source4/scripting/bin/sambaundoguididx
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+import optparse
+import sys
+
+# Find right directory when running from source tree
+sys.path.insert(0, "bin/python")
+
+
+import samba
+import ldb
+import urllib
+import os
+from samba import getopt as options
+from samba.samdb import SamDB
+from samba.dbchecker import dbcheck
+from samba.credentials import Credentials
+parser = optparse.OptionParser("sambaundoguididx")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(options.VersionOptions(parser))
+parser.add_option("-H", "--URL", help="LDB URL for database",
+ type=str, metavar="URL", dest="H")
+opts, args = parser.parse_args()
+
+if len(args) != 0:
+ parser.print_usage()
+ sys.exit(1)
+
+lp_ctx = sambaopts.get_loadparm()
+lp_ctx.set("dsdb:guid index", "false")
+
+if opts.H is None:
+ url = lp_ctx.samdb_url()
+else:
+ url = opts.H
+
+samdb = ldb.Ldb(url=url, options=["modules:"])
+
+partitions = samdb.search(base="@PARTITION",
+ scope=ldb.SCOPE_BASE,
+ attrs=["partition"])
+
+modmsg = ldb.Message()
+modmsg.dn = ldb.Dn(samdb, '@INDEXLIST')
+modmsg.add(ldb.MessageElement(
+ elements=[],
+ flags=ldb.FLAG_MOD_REPLACE,
+ name='@IDXGUID'))
+modmsg.add(ldb.MessageElement(
+ elements=[],
+ flags=ldb.FLAG_MOD_REPLACE,
+ name='@IDX_DN_GUID'))
+
+samdb.transaction_start()
+samdb.modify(modmsg)
+
+privatedir = os.path.dirname(url)
+
+dbs = []
+for part in partitions[0]['partition']:
+ file_quoted = part.split(":")[1]
+ tdbname = urllib.unquote(file_quoted)
+ tdbpath = os.path.join(privatedir, tdbname)
+
+ db = ldb.Ldb(url=tdbpath, options=["modules:"])
+ db.transaction_start()
+ db.modify(modmsg)
+ dbs.append(db)
+
+for db in dbs:
+ db.transaction_commit()
+
+samdb.transaction_commit()
+
+print "Re-opening with the full DB stack"
+samdb = SamDB(url=url,
+ lp=lp_ctx)
+print "Re-triggering another re-index"
+chk = dbcheck(samdb)
+
+chk.reindex_database()
+
+print "Your database has been downgraded to DN-based index values."
+
+print "NOTE: Any use of a Samba 4.8 tool including ldbsearch will auto-upgrade back to GUID index mode"
--
2.11.0
From 0e8a854ba56a6aa7de1d6554b5973705a933d131 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 13 Sep 2017 15:49:43 +1200
Subject: [PATCH 88/88] ldb: Add test showing a search against the index is not
possible
This is not actually a great test, as the filter would
fail to match these anyway, but it at least checks the
codepath is safe.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index b2e31c23f4e..7189e97c303 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -950,6 +950,22 @@ class SearchTests(TestCase):
expression="(&(ou=ouX)(y=a))")
self.assertEqual(len(res11), 0)
+ def test_subtree_and_idx_record(self):
+ """Testing a search against the index record"""
+
+ res11 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(@IDXDN=DC=SAMBA,DC=ORG)")
+ self.assertEqual(len(res11), 0)
+
+ def test_subtree_and_idxone_record(self):
+ """Testing a search against the index record"""
+
+ res11 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(@IDXONE=DC=SAMBA,DC=ORG)")
+ self.assertEqual(len(res11), 0)
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
--
2.11.0
More information about the samba-technical
mailing list