[PATCH] LMDB key value backend for ldb_tdb (to be renamed ldb_key_val)
Gary Lockyer
gary at catalyst.net.nz
Fri Mar 23 01:00:45 UTC 2018
Updated patch set adding an lmdb backend to ldb, incorporating the
feedback received to date. The patch set adds extra testing of the API
and the new key value store layers.
A lot of the names are still tdb specific. Once these patches have
landed on master, I'll put up patches that rename things sensibly.
Please note that these patches can't be pushed until the lib-lmdb and
lmdb-utils have been installed on sndevel.
Comments and reviews greatly appreciated.
Gary.
-------------- next part --------------
From 4fde24d553fd9d299488e6605237178bc8f6920b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:42:17 +1300
Subject: [PATCH 01/49] ldb_wrap: Remove ldb_transaction_cancel_noerr from
ldb_wrap_fork_hook()
Writing to a TDB, without locks (these are per-process) in a forked child is never going to
end well, if a transaction is open at this point we have bigger problems.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 15 +++------------
1 file changed, 3 insertions(+), 12 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 9959b04..8c3bf6f 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -329,20 +329,11 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
}
/*
- when we fork() we need to make sure that any open ldb contexts have
- any open transactions cancelled (ntdb databases doesn't need reopening,
- as we don't use clear_if_first).
- */
+ call tdb_reopen_all() in case there is a TDB open so we are
+ not blocked from re-opening it inside ldb_tdb.
+*/
void ldb_wrap_fork_hook(void)
{
- struct ldb_wrap *w;
-
- for (w=ldb_wrap_list; w; w=w->next) {
- if (ldb_transaction_cancel_noerr(w->ldb) != LDB_SUCCESS) {
- smb_panic("Failed to cancel child transactions\n");
- }
- }
-
if (tdb_reopen_all(1) != 0) {
smb_panic("tdb_reopen_all failed\n");
}
--
2.7.4
From c178a15f82fc09b07aaa3936b91b5a38d35fe4c6 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 15 Mar 2018 13:44:52 +1300
Subject: [PATCH 02/49] ldb_wrap: Remove the magic cache of database handles
except for sam.ldb
sam.ldb is handled in samdb_connect_url(), not this function.
This cache caused issues when "private dir" was changed in a testing script, but also
just generates many-owner shared mutable state that is frowned upon these days.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 8c3bf6f..5325555 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -303,9 +303,13 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
struct ldb_context *ldb;
int ret;
- ldb = ldb_wrap_find(url, ev, lp_ctx, session_info, credentials, flags);
- if (ldb != NULL)
- return talloc_reference(mem_ctx, ldb);
+ /*
+ * Unlike samdb_connect_url() do not try and cache the LDB
+ * handle, get a new one each time. Only sam.ldb is
+ * punitively expensive to open and helpful caches like this
+ * cause challenges (such as if the value for 'private dir'
+ * changes).
+ */
ldb = samba_ldb_init(mem_ctx, ev, lp_ctx, session_info, credentials);
@@ -318,11 +322,6 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
return NULL;
}
- if (!ldb_wrap_add(url, ev, lp_ctx, session_info, credentials, flags, ldb)) {
- talloc_free(ldb);
- return NULL;
- }
-
DEBUG(3,("ldb_wrap open of %s\n", url));
return ldb;
--
2.7.4
From 33d5d8068d9a9f3443f4cb9d878ad6940b28c3f3 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 13:05:55 +1300
Subject: [PATCH 03/49] samba-tool domain classicupgrade: Do not mix
python-samdb transactions and passdb modifications
This worked previously because we knew the same tdb was in use under the hood,
but now that nested TDB transactions are banned this breaks, and it breaks for
LMDB.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/upgrade.py | 48 +++++++++++++++++++-----------------------------
1 file changed, 19 insertions(+), 29 deletions(-)
diff --git a/python/samba/upgrade.py b/python/samba/upgrade.py
index ff47066..e3fa376 100644
--- a/python/samba/upgrade.py
+++ b/python/samba/upgrade.py
@@ -785,38 +785,28 @@ Please fix this account before attempting to upgrade again
result.samdb.transaction_commit()
logger.info("Adding users")
- # Start a new transaction (should speed this up a little, due to index churn)
- result.samdb.transaction_start()
-
- try:
- # Export users to samba4 backend
- logger.info("Importing users")
- for username in userdata:
- if username.lower() == 'administrator':
- if userdata[username].user_sid != dom_sid(str(domainsid) + "-500"):
- logger.error("User 'Administrator' in your existing directory has SID %s, expected it to be %s" % (userdata[username].user_sid, dom_sid(str(domainsid) + "-500")))
- raise ProvisioningError("User 'Administrator' in your existing directory does not have SID ending in -500")
- if username.lower() == 'root':
- if userdata[username].user_sid == dom_sid(str(domainsid) + "-500"):
- logger.warn('User root has been replaced by Administrator')
- else:
- logger.warn('User root has been kept in the directory, it should be removed in favour of the Administrator user')
- s4_passdb.add_sam_account(userdata[username])
- if username in uids:
- add_ad_posix_idmap_entry(result.samdb, userdata[username].user_sid, uids[username], "ID_TYPE_UID", logger)
- if (username in homes) and (homes[username] is not None) and \
- (username in shells) and (shells[username] is not None) and \
- (username in pgids) and (pgids[username] is not None):
- add_posix_attrs(samdb=result.samdb, sid=userdata[username].user_sid, name=username, nisdomain=domainname.lower(), xid_type="ID_TYPE_UID", home=homes[username], shell=shells[username], pgid=pgids[username], logger=logger)
+ # Export users to samba4 backend
+ logger.info("Importing users")
+ for username in userdata:
+ if username.lower() == 'administrator':
+ if userdata[username].user_sid != dom_sid(str(domainsid) + "-500"):
+ logger.error("User 'Administrator' in your existing directory has SID %s, expected it to be %s" % (userdata[username].user_sid, dom_sid(str(domainsid) + "-500")))
+ raise ProvisioningError("User 'Administrator' in your existing directory does not have SID ending in -500")
+ if username.lower() == 'root':
+ if userdata[username].user_sid == dom_sid(str(domainsid) + "-500"):
+ logger.warn('User root has been replaced by Administrator')
+ else:
+ logger.warn('User root has been kept in the directory, it should be removed in favour of the Administrator user')
- except:
- # We need this, so that we do not give even more errors due to not cancelling the transaction
- result.samdb.transaction_cancel()
- raise
+ s4_passdb.add_sam_account(userdata[username])
+ if username in uids:
+ add_ad_posix_idmap_entry(result.samdb, userdata[username].user_sid, uids[username], "ID_TYPE_UID", logger)
+ if (username in homes) and (homes[username] is not None) and \
+ (username in shells) and (shells[username] is not None) and \
+ (username in pgids) and (pgids[username] is not None):
+ add_posix_attrs(samdb=result.samdb, sid=userdata[username].user_sid, name=username, nisdomain=domainname.lower(), xid_type="ID_TYPE_UID", home=homes[username], shell=shells[username], pgid=pgids[username], logger=logger)
- logger.info("Committing 'add users' transaction to disk")
- result.samdb.transaction_commit()
logger.info("Adding users to groups")
# Start a new transaction (should speed this up a little, due to index churn)
--
2.7.4
From 09d59a92ba996ad846956f322f9d3e736dce66de Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 04/49] ldb_mdb: Implement the lmdb backend for ldb
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 710 ++++++++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 57 ++++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
lib/ldb/tools/ldbdump.c | 126 +++++++-
lib/ldb/wscript | 77 ++++-
5 files changed, 966 insertions(+), 5 deletions(-)
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 0000000..f4648e9
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,710 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "lmdb.h"
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX "mdb://"
+#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+
+#define MEGABYTE (1024*1024)
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+ switch (lmdb_err) {
+ case MDB_SUCCESS:
+ return LDB_SUCCESS;
+ case EIO:
+ return LDB_ERR_OPERATIONS_ERROR;
+ case MDB_INCOMPATIBLE:
+ case MDB_CORRUPTED:
+ case MDB_INVALID:
+ return LDB_ERR_UNAVAILABLE;
+ case MDB_BAD_TXN:
+ case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+ case MDB_BAD_DBI:
+#endif
+ case MDB_PANIC:
+ case EINVAL:
+ return LDB_ERR_PROTOCOL_ERROR;
+ case MDB_MAP_FULL:
+ case MDB_DBS_FULL:
+ case MDB_READERS_FULL:
+ case MDB_TLS_FULL:
+ case MDB_TXN_FULL:
+ case EAGAIN:
+ return LDB_ERR_BUSY;
+ case MDB_KEYEXIST:
+ return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ case MDB_NOTFOUND:
+ case ENOENT:
+ return LDB_ERR_NO_SUCH_OBJECT;
+ case EACCES:
+ return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+ default:
+ break;
+ }
+ return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+ int ecode,
+ const char *file,
+ int line)
+{
+ int ldb_err = ldb_mdb_err_map(ecode);
+ char *reason = mdb_strerror(ecode);
+ ldb_asprintf_errstring(ldb,
+ "(%d) - %s at %s:%d",
+ ecode,
+ reason,
+ file,
+ line);
+ return ldb_err;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+ if (ltx == NULL) {
+ return NULL;
+ }
+
+ return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ if (lmdb->txlist) {
+ talloc_steal(lmdb->txlist, ltx);
+ }
+
+ DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ DLIST_REMOVE(lmdb->txlist, ltx);
+ talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx;
+
+ ltx = lmdb->txlist;
+ return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+ MDB_txn *txn;
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
+ }
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ int ret;
+ ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ lmdb->error = ret;
+ ldb_asprintf_errstring(lmdb->ldb,
+ "%s failed: %s\n", __FUNCTION__,
+ mdb_strerror(ret));
+ }
+ lmdb->read_txn = txn;
+ }
+ return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+ TDB_DATA key,
+ TDB_DATA data, int flags)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ int mdb_flags;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ mdb_data.mv_size = data.dsize;
+ mdb_data.mv_data = data.dptr;
+
+ if (flags == TDB_INSERT) {
+ mdb_flags = MDB_NOOVERWRITE;
+ } else if ((flags == TDB_MODIFY)) {
+ /*
+ * Modifying a record, ensure that it exists.
+ * This mimics the TDB semantics
+ */
+ MDB_val value;
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+ if (lmdb->error != MDB_SUCCESS) {
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ mdb_flags = 0;
+ } else {
+ mdb_flags = 0;
+ }
+
+ lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+ ldb_kv_traverse_fn fn,
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+ MDB_cursor *cursor = NULL;
+ int ret;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+ if (lmdb->error != MDB_SUCCESS) {
+ goto done;
+ }
+
+ while ((lmdb->error = mdb_cursor_get(
+ cursor, &mdb_key,
+ &mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+ struct ldb_val key = {
+ .length = mdb_key.mv_size,
+ .data = mdb_key.mv_data,
+ };
+ struct ldb_val data = {
+ .length = mdb_data.mv_size,
+ .data = mdb_data.mv_data,
+ };
+
+ ret = fn(ltdb, key, data, ctx);
+ if (ret != 0) {
+ goto done;
+ }
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ lmdb->error = MDB_SUCCESS;
+ }
+done:
+ if (cursor != NULL) {
+ mdb_cursor_close(cursor);
+ }
+
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+ TDB_DATA key,
+ TDB_DATA key2,
+ TDB_DATA data,
+ void *state)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct TDB_DATA copy;
+ int ret = LDB_SUCCESS;
+
+ /*
+ * Need to take a copy of the data as the delete operation alters the
+ * data, as it is in private lmdb memory.
+ */
+ copy.dsize = data.dsize;
+ copy.dptr = talloc_memdup(ltdb, data.dptr, data.dsize);
+ if (copy.dptr == NULL) {
+ lmdb->error = MDB_PANIC;
+ return ldb_oom(lmdb->ldb);
+ }
+
+ lmdb->error = lmdb_delete(ltdb, key);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to delete %*.*s "
+ "for rekey as %*.*s: %s",
+ (int)key.dsize, (int)key.dsize,
+ (const char *)key.dptr,
+ (int)key2.dsize, (int)key2.dsize,
+ (const char *)key.dptr,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+ lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to rekey %*.*s as %*.*s: %s",
+ (int)key.dsize, (int)key.dsize,
+ (const char *)key.dptr,
+ (int)key2.dsize, (int)key2.dsize,
+ (const char *)key.dptr,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+
+done:
+ if (copy.dptr != NULL) {
+ TALLOC_FREE(copy.dptr);
+ copy.dsize = 0;
+ }
+
+ /*
+ * Explicity invalidate the data, as the delete has done this
+ */
+ data.dsize = 0;
+ data.dptr = NULL;
+
+ return ret;
+}
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
+ int (*parser)(TDB_DATA key, TDB_DATA data,
+ void *private_data),
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ TDB_DATA data;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+ if (lmdb->error != MDB_SUCCESS) {
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ data.dptr = mdb_data.mv_data;
+ data.dsize = mdb_data.mv_size;
+
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+
+ /* We created a read transaction, commit it */
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ lmdb->error = MDB_SUCCESS;
+ if (ltdb->in_transaction == 0 &&
+ ltdb->read_lock_count == 0) {
+ lmdb->error = mdb_txn_begin(lmdb->env,
+ NULL,
+ MDB_RDONLY,
+ &lmdb->read_txn);
+ }
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ ltdb->read_lock_count++;
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+ if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+ }
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct lmdb_trans *ltx;
+ struct lmdb_trans *ltx_head;
+ MDB_txn *tx_parent;
+
+ ltx = talloc_zero(lmdb, struct lmdb_trans);
+ if (ltx == NULL) {
+ return ldb_oom(lmdb->ldb);
+ }
+
+ ltx_head = lmdb_private_trans_head(lmdb);
+
+ tx_parent = lmdb_trans_get_tx(ltx_head);
+
+ lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, <x->tx);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ trans_push(lmdb, ltx);
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+ /* No need to prepare a commit */
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ lmdb->error = mdb_txn_commit(ltx->tx);
+ trans_finished(lmdb, ltx);
+
+ return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+ return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+ return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+ return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+ /*
+ * lmdb does no provide a quick way to determine if the database
+ * has changed. This function always returns true.
+ *
+ * Note that tdb uses a sequence number that allows this function
+ * to be implemented efficiently.
+ */
+ return true;
+}
+
+
+static struct kv_db_ops lmdb_key_value_ops = {
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+ const char *path;
+
+ /* parse the url */
+ if (strchr(url, ':')) {
+ if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+ return NULL;
+ }
+ path = url + MDB_URL_PREFIX_SIZE;
+ } else {
+ path = url;
+ }
+
+ return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx = NULL;
+
+ /*
+ * Close the read transaction if it's open
+ */
+ if (lmdb->read_txn != NULL) {
+ mdb_txn_abort(lmdb->read_txn);
+ }
+
+ if (lmdb->env == NULL) {
+ return 0;
+ }
+
+ /*
+ * Abort any currently active transactions
+ */
+ ltx = lmdb_private_trans_head(lmdb);
+ while (ltx != NULL) {
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ ltx = lmdb_private_trans_head(lmdb);
+ }
+
+ mdb_env_close(lmdb->env);
+ lmdb->env = NULL;
+
+ return 0;
+}
+
+static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path)
+{
+ struct lmdb_private *lmdb;
+ int ret;
+
+ lmdb = talloc_zero(ldb, struct lmdb_private);
+ if (lmdb == NULL) {
+ return NULL;
+ }
+ lmdb->ldb = ldb;
+
+ ret = mdb_env_create(&lmdb->env);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not create MDB environment %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+ ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ if (ret != 0) {
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ mdb_env_set_maxreaders(lmdb->env, 100000);
+ /* MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+ ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not open DB %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ return lmdb;
+
+}
+
+static int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module)
+{
+ const char *path = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct ltdb_private *ltdb = NULL;
+
+ /*
+ * We hold locks, so we must use a private event context
+ * on each returned handle
+ */
+ ldb_set_require_private_event_context(ldb);
+
+ path = lmdb_get_path(url);
+ if (path == NULL) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb = talloc_zero(ldb, struct ltdb_private);
+ if (!ltdb) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ lmdb = lmdb_pvt_create(ldb, ldb, path);
+ if (lmdb == NULL) {
+ ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb->lmdb_private = lmdb;
+ if (flags & LDB_FLG_RDONLY) {
+ ltdb->read_only = true;
+ }
+ return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
+
+
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 0000000..18d28e8
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,57 @@
+/*
+ ldb database library using mdb back end - transaction operations
+
+ Copyright (C) Jakub Hrozek 2015
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include <lmdb.h>
+
+#include "ldb_private.h"
+
+
+struct lmdb_private {
+ struct ldb_context *ldb;
+ MDB_env *env;
+
+ struct lmdb_trans *txlist;
+
+ struct ldb_mdb_metadata {
+ struct ldb_message *attributes;
+ unsigned seqnum;
+ } *meta;
+ int error;
+ MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+ struct lmdb_trans *next;
+ struct lmdb_trans *prev;
+
+ MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 2235bd4..cfabe66 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
struct ltdb_private {
const struct kv_db_ops *kv_ops;
TDB_CONTEXT *tdb;
+ struct lmdb_private *lmdb_private;
unsigned int connect_flags;
unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59..2da2ca8 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
#include <ldb.h>
#include <ldb_private.h>
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
static struct ldb_context *ldb;
bool show_index = false;
bool validate_contents = false;
@@ -166,6 +171,116 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
}
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ int ret;
+ struct MDB_env *env = NULL;
+ struct MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct MDB_cursor *cursor = NULL;
+ struct MDB_val key;
+ struct MDB_val data;
+
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not create MDB environment: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_env_open(env,
+ fname,
+ MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY,
+ 0600);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open environment for %s: (%d) %s\n",
+ fname,
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not start transaction: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open database: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_open(txn, dbi, &cursor);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open cursor: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not find first record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ while (ret != MDB_NOTFOUND) {
+ struct TDB_DATA tkey = {
+ .dptr = key.mv_data,
+ .dsize = key.mv_size
+ };
+ struct TDB_DATA tdata = {
+ .dptr = data.mv_data,
+ .dsize = data.mv_size
+ };
+ traverse_fn(NULL, tkey, tdata, dn);
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not read next record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ }
+ ret = 0;
+
+close_cursor:
+ mdb_cursor_close(cursor);
+close_txn:
+ mdb_txn_commit(txn);
+close_env:
+ mdb_env_close(env);
+
+ if (ret != 0) {
+ return 1;
+ }
+ return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ /* not built with lmdb support */
+ return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
static void usage( void)
{
printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +344,14 @@ static void usage( void)
fname = argv[optind];
- return dump_tdb(fname, dn, emergency);
+ rc = dump_lmdb(fname, dn, emergency);
+ if (rc != 0) {
+ rc = dump_tdb(fname, dn, emergency);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to open %s\n", fname);
+ return 1;
+ }
+ }
+ return 0;
+
}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 1455f92..47e9493 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
srcdir = srcdir + '/..'
sys.path.insert(0, srcdir + '/buildtools/wafsamba')
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -92,6 +92,16 @@ def configure(conf):
implied_deps='replace talloc tdb tevent'):
conf.define('USING_SYSTEM_LDB', 1)
+ if conf.env.standalone_ldb:
+ # Require lmdb support for standalone mode.
+ conf.env.REQUIRE_LMDB = True
+ elif not Options.options.without_ad_dc:
+ # Require lmdb support for addc mode
+ conf.env.REQUIRE_LMDB = True
+ else:
+ conf.env.REQUIRE_LMDB = False
+
+
if conf.CONFIG_SET('USING_SYSTEM_LDB'):
v = VERSION.split('.')
conf.DEFINE('EXPECTED_SYSTEM_LDB_VERSION_MAJOR', int(v[0]))
@@ -110,6 +120,36 @@ def configure(conf):
if not sys.platform.startswith("openbsd"):
conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
+ # if lmdb support is enabled then we require lmdb
+ # is present, build the mdb back end and enable lmdb support in
+ # the tools.
+ if conf.env.REQUIRE_LMDB:
+ if not conf.CHECK_CFG(package='lmdb',
+ args='"lmdb >= 0.9.16" --cflags --libs',
+ msg='Checking for lmdb >= 0.9.16',
+ mandatory=False):
+ if not conf.CHECK_CODE('''
+ #if MDB_VERSION_MAJOR == 0 \
+ && MDB_VERSION_MINOR <= 9 \
+ && MDB_VERSION_PATCH < 16
+ #error LMDB too old
+ #endif
+ ''',
+ 'HAVE_GOOD_LMDB_VERSION',
+ headers='lmdb.h',
+ msg='Checking for lmdb >= 0.9.16 via header check'):
+
+ if conf.env.standalone_ldb:
+ raise Utils.WafError('ldb requires '
+ 'lmdb 0.9.16 or later')
+ elif not Options.options.without_ad_dc:
+ raise Utils.WafError('Samba AD DC requires '
+ 'lmdb 0.9.16 or later')
+
+ if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+ conf.DEFINE('HAVE_LMDB', '1')
+ conf.env.ENABLE_MDB_BACKEND = True
+
conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
conf.SAMBA_CONFIG_H()
@@ -321,6 +361,20 @@ def build(bld):
private_library=True,
deps='tdb ldb')
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_MODULE('ldb_mdb',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ init_function='ldb_mdb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value lmdb',
+ subsystem='ldb')
+ lmdb_deps = ' ldb_mdb_int'
+ else:
+ lmdb_deps = ''
+
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
@@ -338,8 +392,14 @@ def build(bld):
bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ lmdb_deps = ' lmdb'
+ else:
+ lmdb_deps = ''
# ldbdump doesn't get installed
- bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+ bld.SAMBA_BINARY('ldbdump',
+ 'tools/ldbdump.c',
+ deps='ldb-cmdline ldb' + lmdb_deps,
install=False)
bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -353,6 +413,13 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+ source='tests/ldb_mod_op_test.c',
+ cflags='-DTEST_BE=\"mdb\"',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
deps='cmocka ldb',
@@ -385,8 +452,10 @@ def test(ctx):
print("Python testsuite returned %d" % pyret)
cmocka_ret = 0
- for test_exe in ['ldb_tdb_mod_op_test',
- 'ldb_msg_test']:
+ test_exes = ['ldb_tdb_mod_op_test', 'ldb_msg_test']
+ if env.ENABLE_MDB_BACKEND:
+ test_exes.append('ldb_mdb_mod_op_test')
+ for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.7.4
From 55a99ec61d5d7c9dba41840ce70bf254731d4f34 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 05/49] ldb: Handle multiple ldb url schemes
Here we use ldb_relative_path more often and allow for the stripping of
ldb:// and mdb://.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 6 ++++++
source4/dsdb/samdb/ldb_modules/partition_metadata.c | 20 +++++++++-----------
source4/dsdb/samdb/ldb_modules/schema_load.c | 19 +++++--------------
3 files changed, 20 insertions(+), 25 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 5325555..1b12d64 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -351,6 +351,12 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
if (strncmp("tdb://", base_url, 6) == 0) {
base_url = base_url+6;
}
+ else if (strncmp("mdb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
+ else if (strncmp("ldb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
path = talloc_strdup(mem_ctx, base_url);
if (path == NULL) {
return NULL;
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index e3ad0d8..510228e 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -18,6 +18,7 @@
*/
#include "dsdb/samdb/ldb_modules/partition.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
#define LDB_METADATA_SEQ_NUM "SEQ_NUM"
@@ -185,7 +186,6 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
TALLOC_CTX *tmp_ctx;
struct partition_private_data *data;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename, *dirname;
int open_flags;
struct stat statbuf;
@@ -202,15 +202,11 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
+
if (!filename) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
@@ -222,7 +218,9 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
/* While provisioning, sam.ldb.d directory may not exist,
* so create it. Ignore errors, if it already exists. */
- dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name);
+ dirname = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d");
if (!dirname) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 2099fac..e21a5a5 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -32,6 +32,7 @@
#include <tdb.h>
#include "lib/tdb_wrap/tdb_wrap.h"
#include "dsdb/samdb/ldb_modules/util.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
struct schema_load_private_data {
@@ -58,7 +59,6 @@ static int schema_metadata_open(struct ldb_module *module)
struct ldb_context *ldb = ldb_module_get_ctx(module);
TALLOC_CTX *tmp_ctx;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename;
int open_flags;
struct stat statbuf;
@@ -74,19 +74,10 @@ static int schema_metadata_open(struct ldb_module *module)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
- if (!filename) {
- talloc_free(tmp_ctx);
- return ldb_oom(ldb);
- }
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
open_flags = O_RDWR;
if (stat(filename, &statbuf) != 0) {
--
2.7.4
From 84826c8f8bc2c54e472be39ef883b5490ff2a869 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 06/49] ldb: Introduce new ldb:// prefix to allow failover
If you try to open with tdb first you find that you can clobber a
database due to the default tdb behaviour.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_ldb/ldb_ldb.c | 78 +++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.c | 79 +++++++++++++++++++++++++-----------------
lib/ldb/ldb_mdb/ldb_mdb.h | 3 ++
lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 +++++++++++++++++
lib/ldb/wscript | 19 ++++++++--
5 files changed, 176 insertions(+), 34 deletions(-)
create mode 100644 lib/ldb/ldb_ldb/ldb_ldb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c
diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
new file mode 100644
index 0000000..a63967f
--- /dev/null
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -0,0 +1,78 @@
+/*
+ * ldb connection and module initialisation
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#include "ldb_private.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+
+/*
+ connect to the database
+*/
+static int lldb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **module)
+{
+ const char *path;
+ int ret;
+
+ /*
+ * Check and remove the url prefix
+ */
+ if (strchr(url, ':')) {
+ if (strncmp(url, "ldb://", 6) != 0) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR,
+ "Invalid ldb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ path = url+6;
+ } else {
+ path = url;
+ }
+
+ /*
+ * Don't create the database if it's not there
+ */
+ flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+ /*
+ * Try opening the database as an lmdb
+ */
+ ret = lmdb_connect(ldb, path, flags, options, module);
+ if (ret == LDB_SUCCESS) {
+ return ret;
+ }
+ if (ret == LDB_ERR_UNAVAILABLE) {
+ /*
+ * Not mdb so try as tdb
+ */
+ ret = ltdb_connect(ldb, path, flags, options, module);
+ }
+#else
+ ret = ltdb_connect(ldb, path, flags, options, module);
+#endif
+ return ret;
+}
+
+int ldb_ldb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("ldb", lldb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f4648e9..0029136 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -613,26 +613,30 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
return 0;
}
-static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
- struct ldb_context *ldb,
- const char *path)
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags,
+ struct lmdb_private *lmdb)
{
- struct lmdb_private *lmdb;
int ret;
+ unsigned int mdb_flags;
- lmdb = talloc_zero(ldb, struct lmdb_private);
- if (lmdb == NULL) {
- return NULL;
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
}
- lmdb->ldb = ldb;
ret = mdb_env_create(&lmdb->env);
if (ret != 0) {
- ldb_asprintf_errstring(ldb,
- "Could not create MDB environment %s: %s\n",
- path, mdb_strerror(ret));
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
}
/* Close when lmdb is released */
@@ -640,34 +644,45 @@ static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
if (ret != 0) {
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not open MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return ldb_mdb_err_map(ret);
}
mdb_env_set_maxreaders(lmdb->env, 100000);
/* MDB_NOSUBDIR implies there is a separate file called path and a
* separate lockfile called path-lock
*/
- ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ if (flags & LDB_FLG_RDONLY) {
+ mdb_flags |= MDB_RDONLY;
+ }
+ ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
talloc_free(lmdb);
- return NULL;
+ return ldb_mdb_err_map(ret);
}
- return lmdb;
+ return LDB_SUCCESS;
}
-static int lmdb_connect(struct ldb_context *ldb, const char *url,
- unsigned int flags, const char *options[],
- struct ldb_module **_module)
+int lmdb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **_module)
{
const char *path = NULL;
struct lmdb_private *lmdb = NULL;
struct ltdb_private *ltdb = NULL;
+ int ret;
/*
* We hold locks, so we must use a private event context
@@ -686,12 +701,19 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
}
- ltdb->kv_ops = &lmdb_key_value_ops;
- lmdb = lmdb_pvt_create(ldb, ldb, path);
+ lmdb = talloc_zero(ldb, struct lmdb_private);
if (lmdb == NULL) {
- ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
- return LDB_ERR_OPERATIONS_ERROR;
+ TALLOC_FREE(ltdb);
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ lmdb->ldb = ldb;
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ if (ret != LDB_SUCCESS) {
+ return ret;
}
ltdb->lmdb_private = lmdb;
@@ -701,10 +723,3 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
-
-
-int ldb_mdb_init(const char *version)
-{
- LDB_MODULE_CHECK_VERSION(version);
- return ldb_register_backend("mdb", lmdb_connect, false);
-}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 18d28e8..e62e353 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -53,5 +53,8 @@ struct lmdb_trans {
};
int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module);
#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 0000000..339c3f2
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 47e9493..4c32e95 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -364,17 +364,32 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_MODULE('ldb_mdb',
bld.SUBDIR('ldb_mdb',
- '''ldb_mdb.c '''),
+ '''ldb_mdb_init.c'''),
init_function='ldb_mdb_init',
module_init_name='ldb_init_module',
internal_module=False,
- deps='ldb ldb_key_value lmdb',
+ deps='ldb ldb_key_value ldb_mdb_int',
subsystem='ldb')
+
+ bld.SAMBA_LIBRARY('ldb_mdb_int',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ private_library=True,
+ deps='ldb lmdb ldb_key_value')
lmdb_deps = ' ldb_mdb_int'
else:
lmdb_deps = ''
+ bld.SAMBA_MODULE('ldb_ldb',
+ bld.SUBDIR('ldb_ldb',
+ '''ldb_ldb.c'''),
+ init_function='ldb_ldb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value' + lmdb_deps,
+ subsystem='ldb')
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
--
2.7.4
From 80868f633e3f490fbf568de9be1f451d509372c2 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 5 Jan 2018 09:28:54 +1300
Subject: [PATCH 07/49] tests: Replace some references to tdb with ldb://
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/tests/encrypted_secrets.py | 2 +-
source4/torture/drs/python/samba_tool_drs.py | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/python/samba/tests/encrypted_secrets.py b/python/samba/tests/encrypted_secrets.py
index 3b6934f..114f3b9 100644
--- a/python/samba/tests/encrypted_secrets.py
+++ b/python/samba/tests/encrypted_secrets.py
@@ -54,7 +54,7 @@ class EncryptedSecretsTests(TestCase):
backend_subpath = os.path.join("sam.ldb.d",
backend_filename)
backend_path = self.lp.private_path(backend_subpath)
- backenddb = ldb.Ldb(backend_path)
+ backenddb = ldb.Ldb("ldb://" + backend_path, flags=ldb.FLG_DONT_CREATE_DB)
dn = "CN=Administrator,CN=Users,%s" % basedn
diff --git a/source4/torture/drs/python/samba_tool_drs.py b/source4/torture/drs/python/samba_tool_drs.py
index fcc681d..502c809 100644
--- a/source4/torture/drs/python/samba_tool_drs.py
+++ b/source4/torture/drs/python/samba_tool_drs.py
@@ -282,7 +282,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
@@ -291,7 +291,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertEqual(ds_name, server_ds_name)
self.assertEqual(ldap_service_name, server_ldap_service_name)
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
def get_krbtgt_pw():
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
@@ -346,13 +346,13 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
config_nc_name = ldb_rootdse["configurationNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
self.assertIsNotNone(krbtgt_pw)
@@ -381,7 +381,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertRaises(samba.tests.BlackboxProcessError, demote_self)
# While we have this cloned, try demoting the other server on the clone
- out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H %s/private/sam.ldb"
+ out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H ldb://%s/private/sam.ldb"
% (self.dc2,
self.tempdir))
--
2.7.4
From ec2a04f068e975ec296834fe4c160b6b6a7ffe60 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 12:40:01 +1300
Subject: [PATCH 08/49] tests/dlz_bind9: support for multiple db types
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
source4/torture/dns/dlz_bind9.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/torture/dns/dlz_bind9.c b/source4/torture/dns/dlz_bind9.c
index 2234e7a..42b104e 100644
--- a/source4/torture/dns/dlz_bind9.c
+++ b/source4/torture/dns/dlz_bind9.c
@@ -58,7 +58,7 @@ static char *test_dlz_bind9_binddns_dir(struct torture_context *tctx,
const char *file)
{
return talloc_asprintf(tctx,
- "%s/%s",
+ "ldb://%s/%s",
lpcfg_binddns_dir(tctx->lp_ctx),
file);
}
--
2.7.4
From 823f4256a105278d35fcd1353f2e2b02c31f80c9 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 14:38:19 +1300
Subject: [PATCH 09/49] provision: allow provisioning of a different database
backend
This sets the backendStore field in @PARTITION, depending on which
argument you set in the provision.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/netcmd/domain.py | 14 +++++++++---
python/samba/provision/__init__.py | 38 +++++++++++++++++++++++----------
python/samba/provision/sambadns.py | 35 ++++++++++++++++++++++--------
python/samba/samdb.py | 3 +++
source4/setup/provision_partitions.ldif | 1 +
5 files changed, 68 insertions(+), 23 deletions(-)
diff --git a/python/samba/netcmd/domain.py b/python/samba/netcmd/domain.py
index d3b9b0c..efb0780 100644
--- a/python/samba/netcmd/domain.py
+++ b/python/samba/netcmd/domain.py
@@ -42,7 +42,7 @@ from samba.net import Net, LIBNET_JOIN_AUTOMATIC
import samba.ntacls
from samba.join import join_RODC, join_DC, join_subdomain
from samba.auth import system_session
-from samba.samdb import SamDB
+from samba.samdb import SamDB, get_default_backend_store
from samba.ndr import ndr_unpack, ndr_pack, ndr_print
from samba.dcerpc import drsuapi
from samba.dcerpc import drsblobs
@@ -257,6 +257,10 @@ class cmd_domain_provision(Command):
Option("--plaintext-secrets", action="store_true",
help="Store secret/sensitive values as plain text on disk" +
"(default is to encrypt secret/ensitive values)"),
+ Option("--backend-store", type="choice", metavar="BACKENDSTORE",
+ choices=["tdb", "mdb"],
+ help="Specify the database backend to be used "
+ "(default is %s)" % get_default_backend_store()),
]
openldap_options = [
@@ -327,7 +331,8 @@ class cmd_domain_provision(Command):
ldap_backend_forced_uri=None,
ldap_dryrun_mode=None,
base_schema=None,
- plaintext_secrets=False):
+ plaintext_secrets=False,
+ backend_store=None):
self.logger = self.get_logger("provision")
if quiet:
@@ -475,6 +480,8 @@ class cmd_domain_provision(Command):
domain_sid = security.dom_sid(domain_sid)
session = system_session()
+ if backend_store is None:
+ backend_store = get_default_backend_store()
try:
result = provision(self.logger,
session, smbconf=smbconf, targetdir=targetdir,
@@ -497,7 +504,8 @@ class cmd_domain_provision(Command):
ldap_backend_forced_uri=ldap_backend_forced_uri,
nosync=ldap_backend_nosync, ldap_dryrun_mode=ldap_dryrun_mode,
base_schema=base_schema,
- plaintext_secrets=plaintext_secrets)
+ plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
except ProvisioningError as e:
raise CommandError("Provision failed", e)
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index f36f277..aaf839c 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -123,6 +123,7 @@ from samba.schema import Schema
from samba.samdb import SamDB
from samba.dbchecker import dbcheck
from samba.provision.kerberos import create_kdc_conf
+from samba.samdb import get_default_backend_store
DEFAULT_POLICY_GUID = "31B2F340-016D-11D2-945F-00C04FB984F9"
DEFAULT_DC_POLICY_GUID = "6AC1786C-016F-11D2-945F-00C04FB984F9"
@@ -814,7 +815,8 @@ def setup_name_mappings(idmap, sid, root_uid, nobody_uid,
def setup_samdb_partitions(samdb_path, logger, lp, session_info,
provision_backend, names, serverrole,
- erase=False, plaintext_secrets=False):
+ erase=False, plaintext_secrets=False,
+ backend_store=None):
"""Setup the partitions for the SAM database.
Alternatively, provision() may call this, and then populate the database.
@@ -847,11 +849,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
if not plaintext_secrets:
required_features = "requiredFeatures: encryptedSecrets"
+ if backend_store is None:
+ backend_store = get_default_backend_store()
+ backend_store_line = "backendStore: %s" % backend_store
+
samdb.transaction_start()
try:
logger.info("Setting up sam.ldb partitions and settings")
setup_add_ldif(samdb, setup_path("provision_partitions.ldif"), {
- "LDAP_BACKEND_LINE": ldap_backend_line
+ "LDAP_BACKEND_LINE": ldap_backend_line,
+ "BACKEND_STORE": backend_store_line
})
@@ -1245,7 +1252,7 @@ def create_default_gpo(sysvolpath, dnsdomain, policyguid, policyguid_dc):
def setup_samdb(path, session_info, provision_backend, lp, names,
logger, fill, serverrole, schema, am_rodc=False,
- plaintext_secrets=False):
+ plaintext_secrets=False, backend_store=None):
"""Setup a complete SAM Database.
:note: This will wipe the main SAM database file!
@@ -1254,7 +1261,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
# Also wipes the database
setup_samdb_partitions(path, logger=logger, lp=lp,
provision_backend=provision_backend, session_info=session_info,
- names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets)
+ names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
# Load the database, but don's load the global schema and don't connect
# quite yet
@@ -1293,7 +1301,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
def fill_samdb(samdb, lp, names, logger, policyguid,
policyguid_dc, fill, adminpass, krbtgtpass, machinepass, dns_backend,
dnspass, invocationid, ntdsguid, serverrole, am_rodc=False,
- dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None):
+ dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None,
+ backend_store=None):
if next_rid is None:
next_rid = 1000
@@ -1831,7 +1840,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
invocationid=None, machinepass=None, ntdsguid=None,
dns_backend=None, dnspass=None,
serverrole=None, dom_for_fun_level=None,
- am_rodc=False, lp=None, use_ntvfs=False, skip_sysvolacl=False):
+ am_rodc=False, lp=None, use_ntvfs=False,
+ skip_sysvolacl=False, backend_store=None):
# create/adapt the group policy GUIDs
# Default GUID for default policy are described at
# "How Core Group Policy Works"
@@ -1863,7 +1873,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
dns_backend=dns_backend, dnspass=dnspass,
ntdsguid=ntdsguid, serverrole=serverrole,
dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
- next_rid=next_rid, dc_rid=dc_rid)
+ next_rid=next_rid, dc_rid=dc_rid,
+ backend_store=backend_store)
# Set up group policies (domain policy and domain controller
# policy)
@@ -1913,7 +1924,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
setup_ad_dns(samdb, secrets_ldb, names, paths, lp, logger,
hostip=hostip, hostip6=hostip6, dns_backend=dns_backend,
dnspass=dnspass, os_level=dom_for_fun_level,
- targetdir=targetdir, fill_level=samdb_fill)
+ targetdir=targetdir, fill_level=samdb_fill,
+ backend_store=backend_store)
domainguid = samdb.searchone(basedn=samdb.get_default_basedn(),
attribute="objectGUID")
@@ -2033,7 +2045,7 @@ def provision(logger, session_info, smbconf=None,
use_rfc2307=False, maxuid=None, maxgid=None, skip_sysvolacl=True,
ldap_backend_forced_uri=None, nosync=False, ldap_dryrun_mode=False,
ldap_backend_extra_port=None, base_schema=None,
- plaintext_secrets=False):
+ plaintext_secrets=False, backend_store=None):
"""Provision samba4
:note: caution, this wipes all existing data!
@@ -2050,6 +2062,8 @@ def provision(logger, session_info, smbconf=None,
if backend_type is None:
backend_type = "ldb"
+ if backend_store is None:
+ backend_store = get_default_backend_store()
if domainsid is None:
domainsid = security.random_sid()
@@ -2233,7 +2247,8 @@ def provision(logger, session_info, smbconf=None,
provision_backend, lp, names, logger=logger,
serverrole=serverrole,
schema=schema, fill=samdb_fill, am_rodc=am_rodc,
- plaintext_secrets=plaintext_secrets)
+ plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
if serverrole == "active directory domain controller":
if paths.netlogon is None:
@@ -2264,7 +2279,8 @@ def provision(logger, session_info, smbconf=None,
dnspass=dnspass, serverrole=serverrole,
dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
lp=lp, use_ntvfs=use_ntvfs,
- skip_sysvolacl=skip_sysvolacl)
+ skip_sysvolacl=skip_sysvolacl,
+ backend_store=backend_store)
if not is_heimdal_built():
create_kdc_conf(paths.kdcconf, realm, domain, os.path.dirname(lp.get("log file")))
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index 4cc15b0..ce1b769 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -29,6 +29,7 @@ from base64 import b64encode
import subprocess
import samba
from samba.tdb_util import tdb_copy
+from samba.mdb_util import mdb_copy
from samba.ndr import ndr_pack, ndr_unpack
from samba import setup_file
from samba.dcerpc import dnsp, misc, security
@@ -58,6 +59,7 @@ from samba.provision.common import (
FILL_DRS,
)
+from samba.samdb import get_default_backend_store
def get_domainguid(samdb, domaindn):
res = samdb.search(base=domaindn, scope=ldb.SCOPE_BASE, attrs=["objectGUID"])
@@ -787,12 +789,19 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
# Find the partitions and corresponding filenames
partfile = {}
- res = samdb.search(base="@PARTITION", scope=ldb.SCOPE_BASE, attrs=["partition"])
+ res = samdb.search(base="@PARTITION",
+ scope=ldb.SCOPE_BASE,
+ attrs=["partition", "backendStore"])
for tmp in res[0]["partition"]:
(nc, fname) = tmp.split(':')
partfile[nc.upper()] = fname
+ backend_store = get_default_backend_store()
+ if "backendStore" in res[0]:
+ backend_store = res[0]["backendStore"][0]
+
# Create empty domain partition
+
domaindn = names.domaindn.upper()
domainpart_file = os.path.join(dns_dir, partfile[domaindn])
try:
@@ -800,7 +809,8 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
file(domainpart_file, 'w').close()
# Fill the basedn and @OPTION records in domain partition
- dom_ldb = samba.Ldb(domainpart_file)
+ dom_url = "%s://%s" % (backend_store, domainpart_file)
+ dom_ldb = samba.Ldb(dom_url)
domainguid_line = "objectGUID: %s\n-" % domainguid
descr = b64encode(get_domain_descriptor(domainsid))
setup_add_ldif(dom_ldb, setup_path("provision_basedn.ldif"), {
@@ -859,8 +869,12 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
os.path.join(dns_dir, "sam.ldb"))
for nc in partfile:
pfile = partfile[nc]
- tdb_copy(os.path.join(private_dir, pfile),
- os.path.join(dns_dir, pfile))
+ if backend_store == "mdb":
+ mdb_copy(os.path.join(private_dir, pfile),
+ os.path.join(dns_dir, pfile))
+ else:
+ tdb_copy(os.path.join(private_dir, pfile),
+ os.path.join(dns_dir, pfile))
except:
logger.error(
"Failed to setup database for BIND, AD based DNS cannot be used")
@@ -875,7 +889,7 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
os.chown(dpath, -1, paths.bind_gid)
os.chmod(dpath, 0o770)
for f in files:
- if f.endswith('.ldb') or f.endswith('.tdb'):
+ if f.endswith(('.ldb', '.tdb', 'ldb-lock')):
fpath = os.path.join(dirname, f)
os.chown(fpath, -1, paths.bind_gid)
os.chmod(fpath, 0o660)
@@ -1069,7 +1083,7 @@ def fill_dns_data_partitions(samdb, domainsid, site, domaindn, forestdn,
def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, dnspass=None, hostip=None, hostip6=None,
- targetdir=None, fill_level=FILL_FULL):
+ targetdir=None, fill_level=FILL_FULL, backend_store=None):
"""Provision DNS information (assuming GC role)
:param samdb: LDB object connected to sam.ldb file
@@ -1164,12 +1178,14 @@ def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
if dns_backend.startswith("BIND9_"):
setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, site=site, dnspass=dnspass, hostip=hostip,
- hostip6=hostip6, targetdir=targetdir)
+ hostip6=hostip6, targetdir=targetdir,
+ backend_store=backend_store)
def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, site=None, dnspass=None, hostip=None,
- hostip6=None, targetdir=None, key_version_number=None):
+ hostip6=None, targetdir=None, key_version_number=None,
+ backend_store=None):
"""Provision DNS information (assuming BIND9 backend in DC role)
:param samdb: LDB object connected to sam.ldb file
@@ -1216,7 +1232,8 @@ def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
ntdsguid=names.ntdsguid)
if dns_backend == "BIND9_DLZ" and os_level >= DS_DOMAIN_FUNCTION_2003:
- create_samdb_copy(samdb, logger, paths, names, names.domainsid, domainguid)
+ create_samdb_copy(samdb, logger, paths,
+ names, names.domainsid, domainguid)
create_named_conf(paths, realm=names.realm,
dnsdomain=names.dnsdomain, dns_backend=dns_backend,
diff --git a/python/samba/samdb.py b/python/samba/samdb.py
index 348bd21..d7faa23 100644
--- a/python/samba/samdb.py
+++ b/python/samba/samdb.py
@@ -37,6 +37,9 @@ from samba.dcerpc import security
__docformat__ = "restructuredText"
+def get_default_backend_store():
+ return "tdb"
+
class SamDB(samba.Ldb):
"""The SAM database."""
diff --git a/source4/setup/provision_partitions.ldif b/source4/setup/provision_partitions.ldif
index 728f327..4cd5794 100644
--- a/source4/setup/provision_partitions.ldif
+++ b/source4/setup/provision_partitions.ldif
@@ -2,5 +2,6 @@ dn: @PARTITION
replicateEntries: @ATTRIBUTES
replicateEntries: @INDEXLIST
replicateEntries: @OPTIONS
+${BACKEND_STORE}
${LDAP_BACKEND_LINE}
--
2.7.4
From 589acf5f403730a3c6a5524f96626e8188b2d42b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 10:58:11 +1300
Subject: [PATCH 10/49] provision: Set @INDEXLIST first when building dummy
sam.ldb
The new LMDB backed will not allow normal records to be added before the @INDEXLIST
as this is what forces the GUID index mode.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
python/samba/provision/sambadns.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index ce1b769..7a85546 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -811,6 +811,11 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
# Fill the basedn and @OPTION records in domain partition
dom_url = "%s://%s" % (backend_store, domainpart_file)
dom_ldb = samba.Ldb(dom_url)
+
+ # We need the dummy main-domain DB to have the correct @INDEXLIST
+ index_res = samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE)
+ dom_ldb.add(index_res[0])
+
domainguid_line = "objectGUID: %s\n-" % domainguid
descr = b64encode(get_domain_descriptor(domainsid))
setup_add_ldif(dom_ldb, setup_path("provision_basedn.ldif"), {
@@ -821,9 +826,6 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
setup_add_ldif(dom_ldb,
setup_path("provision_basedn_options.ldif"), None)
- # We need the dummy main-domain DB to have the correct @INDEXLIST
- index_res = samdb.search(base="@INDEXLIST", scope=ldb.SCOPE_BASE)
- dom_ldb.add(index_res[0])
except:
logger.error(
"Failed to setup database for BIND, AD based DNS cannot be used")
--
2.7.4
From f3471e998c62d202461b189b2c67016e698c33e5 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Feb 2018 11:10:34 +1300
Subject: [PATCH 11/49] dsdb: add lmdbLevelOne as a required feature.
---
python/samba/provision/__init__.py | 12 +++++++++++-
source4/dsdb/samdb/ldb_modules/samba_dsdb.c | 5 ++++-
source4/dsdb/samdb/samdb.h | 9 +++++++++
3 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index aaf839c..dcdb2b7 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -845,7 +845,7 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
if provision_backend.type != "ldb":
ldap_backend_line = "ldapBackend: %s" % provision_backend.ldap_uri
- required_features = "# No required features"
+ required_features = None
if not plaintext_secrets:
required_features = "requiredFeatures: encryptedSecrets"
@@ -853,6 +853,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
backend_store = get_default_backend_store()
backend_store_line = "backendStore: %s" % backend_store
+ if backend_store == "mdb":
+ if required_features is not None:
+ required_features += "\n"
+ else:
+ required_features = ""
+ required_features += "requiredFeatures: lmdbLevelOne"
+
+ if required_features is None:
+ required_features = "# No required features"
+
samdb.transaction_start()
try:
logger.info("Setting up sam.ldb partitions and settings")
diff --git a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
index e0acb4e..2605c1e 100644
--- a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
+++ b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
@@ -234,8 +234,11 @@ static bool check_required_features(struct ldb_message_element *el)
int k;
DATA_BLOB esf = data_blob_string_const(
SAMBA_ENCRYPTED_SECRETS_FEATURE);
+ DATA_BLOB lmdbl1 = data_blob_string_const(
+ SAMBA_LMDB_LEVEL_ONE_FEATURE);
for (k = 0; k < el->num_values; k++) {
- if (data_blob_cmp(&esf, &el->values[k]) != 0) {
+ if ((data_blob_cmp(&esf, &el->values[k]) != 0) &&
+ (data_blob_cmp(&lmdbl1, &el->values[k]) != 0)) {
return false;
}
}
diff --git a/source4/dsdb/samdb/samdb.h b/source4/dsdb/samdb/samdb.h
index eb52740..a095858 100644
--- a/source4/dsdb/samdb/samdb.h
+++ b/source4/dsdb/samdb/samdb.h
@@ -342,5 +342,14 @@ struct dsdb_extended_sec_desc_propagation_op {
#define SAMBA_SORTED_LINKS_FEATURE "sortedLinks"
#define SAMBA_ENCRYPTED_SECRETS_FEATURE "encryptedSecrets"
+/*
+ * lmdb level one feature is an experimental release with basic support
+ * for lmdb database files, instead of tdb.
+ * - Keys are limited to 511 bytes long so GUID indexes are required
+ * - Currently only the:
+ * partition data files
+ * are in lmdb format.
+ */
+#define SAMBA_LMDB_LEVEL_ONE_FEATURE "lmdbLevelOne"
#endif /* __SAMDB_H__ */
--
2.7.4
From 80fc0a3a549aff4a75a3b88ebbb98b3339b19049 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 16 Feb 2018 17:13:26 +1300
Subject: [PATCH 12/49] ldb: Change some prototypes to using ldb_val instead of
TDB_DATA
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 59 +++++++++++++++++++++++----------------------
lib/ldb/ldb_tdb/ldb_index.c | 8 +++---
lib/ldb/ldb_tdb/ldb_tdb.c | 49 +++++++++++++++++++++++++++++++------
lib/ldb/ldb_tdb/ldb_tdb.h | 8 +++---
4 files changed, 79 insertions(+), 45 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 0029136..f05b11b 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -145,8 +145,8 @@ static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
}
static int lmdb_store(struct ltdb_private *ltdb,
- TDB_DATA key,
- TDB_DATA data, int flags)
+ struct ldb_val key,
+ struct ldb_val data, int flags)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
MDB_val mdb_key;
@@ -167,11 +167,11 @@ static int lmdb_store(struct ltdb_private *ltdb,
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
- mdb_data.mv_size = data.dsize;
- mdb_data.mv_data = data.dptr;
+ mdb_data.mv_size = data.length;
+ mdb_data.mv_data = data.data;
if (flags == TDB_INSERT) {
mdb_flags = MDB_NOOVERWRITE;
@@ -202,7 +202,8 @@ static int lmdb_store(struct ltdb_private *ltdb,
return ldb_mdb_err_map(lmdb->error);
}
-static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
MDB_val mdb_key;
@@ -221,8 +222,8 @@ static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
if (lmdb->error != MDB_SUCCESS) {
@@ -298,22 +299,22 @@ done:
}
static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
- TDB_DATA key,
- TDB_DATA key2,
- TDB_DATA data,
+ struct ldb_val key,
+ struct ldb_val key2,
+ struct ldb_val data,
void *state)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
- struct TDB_DATA copy;
+ struct ldb_val copy;
int ret = LDB_SUCCESS;
/*
* Need to take a copy of the data as the delete operation alters the
* data, as it is in private lmdb memory.
*/
- copy.dsize = data.dsize;
- copy.dptr = talloc_memdup(ltdb, data.dptr, data.dsize);
- if (copy.dptr == NULL) {
+ copy.length = data.length;
+ copy.data = talloc_memdup(ltdb, data.data, data.length);
+ if (copy.data == NULL) {
lmdb->error = MDB_PANIC;
return ldb_oom(lmdb->ldb);
}
@@ -325,10 +326,10 @@ static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
LDB_DEBUG_ERROR,
"Failed to delete %*.*s "
"for rekey as %*.*s: %s",
- (int)key.dsize, (int)key.dsize,
- (const char *)key.dptr,
- (int)key2.dsize, (int)key2.dsize,
- (const char *)key.dptr,
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
mdb_strerror(lmdb->error));
ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
goto done;
@@ -339,26 +340,26 @@ static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
lmdb->ldb,
LDB_DEBUG_ERROR,
"Failed to rekey %*.*s as %*.*s: %s",
- (int)key.dsize, (int)key.dsize,
- (const char *)key.dptr,
- (int)key2.dsize, (int)key2.dsize,
- (const char *)key.dptr,
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
mdb_strerror(lmdb->error));
ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
goto done;
}
done:
- if (copy.dptr != NULL) {
- TALLOC_FREE(copy.dptr);
- copy.dsize = 0;
+ if (copy.data != NULL) {
+ TALLOC_FREE(copy.data);
+ copy.length = 0;
}
/*
* Explicity invalidate the data, as the delete has done this
*/
- data.dsize = 0;
- data.dptr = NULL;
+ data.length = 0;
+ data.data = NULL;
return ret;
}
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index bb534c3..076db10 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2783,11 +2783,11 @@ static int re_key(struct ltdb_private *ltdb, struct ldb_val ldb_key, struct ldb_
}
if (key.dsize != key2.dsize ||
(memcmp(key.dptr, key2.dptr, key.dsize) != 0)) {
- TDB_DATA data = {
- .dptr = val.data,
- .dsize = val.length
+ struct ldb_val ldb_key2 = {
+ .data = key2.dptr,
+ .length = key2.dsize
};
- ltdb->kv_ops->update_in_iterate(ltdb, key, key2, data, ctx);
+ ltdb->kv_ops->update_in_iterate(ltdb, ldb_key, ldb_key2, val, ctx);
}
talloc_free(key2.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 9c8c771..1e8bd29 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -413,8 +413,17 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
return ret;
}
-static int ltdb_tdb_store(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags)
+static int ltdb_tdb_store(struct ltdb_private *ltdb, struct ldb_val ldb_key,
+ struct ldb_val ldb_data, int flags)
{
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+ TDB_DATA data = {
+ .dptr = ldb_data.data,
+ .dsize = ldb_data.length
+ };
return tdb_store(ltdb->tdb, key, data, flags);
}
@@ -435,7 +444,8 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- TDB_DATA tdb_key, tdb_data;
+ TDB_DATA tdb_key;
+ struct ldb_val ldb_key;
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -461,10 +471,10 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
return LDB_ERR_OTHER;
}
- tdb_data.dptr = ldb_data.data;
- tdb_data.dsize = ldb_data.length;
+ ldb_key.data = tdb_key.dptr;
+ ldb_key.length = tdb_key.dsize;
- ret = ltdb->kv_ops->store(ltdb, tdb_key, tdb_data, flgs);
+ ret = ltdb->kv_ops->store(ltdb, ldb_key, ldb_data, flgs);
if (ret != 0) {
bool is_special = ldb_dn_is_special(msg->dn);
ret = ltdb->kv_ops->error(ltdb);
@@ -650,8 +660,12 @@ static int ltdb_add(struct ltdb_context *ctx)
return ret;
}
-static int ltdb_tdb_delete(struct ltdb_private *ltdb, TDB_DATA tdb_key)
+static int ltdb_tdb_delete(struct ltdb_private *ltdb, struct ldb_val ldb_key)
{
+ TDB_DATA tdb_key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
return tdb_delete(ltdb->tdb, tdb_key);
}
@@ -664,6 +678,7 @@ int ltdb_delete_noindex(struct ldb_module *module,
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct ldb_val ldb_key;
TDB_DATA tdb_key;
int ret;
TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -682,7 +697,10 @@ int ltdb_delete_noindex(struct ldb_module *module,
return LDB_ERR_OTHER;
}
- ret = ltdb->kv_ops->delete(ltdb, tdb_key);
+ ldb_key.data = tdb_key.dptr;
+ ldb_key.length = tdb_key.dsize;
+
+ ret = ltdb->kv_ops->delete(ltdb, ldb_key);
TALLOC_FREE(tdb_key_ctx);
if (ret != 0) {
@@ -1747,12 +1765,27 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
}
}
-static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA key2, TDB_DATA data, void *state)
+static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
+ struct ldb_val ldb_key,
+ struct ldb_val ldb_key2,
+ struct ldb_val ldb_data, void *state)
{
int tdb_ret;
struct ldb_context *ldb;
struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
struct ldb_module *module = ctx->module;
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+ TDB_DATA key2 = {
+ .dptr = ldb_key2.data,
+ .dsize = ldb_key2.length
+ };
+ TDB_DATA data = {
+ .dptr = ldb_data.data,
+ .dsize = ldb_data.length
+ };
ldb = ldb_module_get_ctx(module);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index cfabe66..92bbd0e 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -10,11 +10,11 @@ typedef int (*ldb_kv_traverse_fn)(struct ltdb_private *ltdb,
void *ctx);
struct kv_db_ops {
- int (*store)(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags);
- int (*delete)(struct ltdb_private *ltdb, TDB_DATA key);
+ int (*store)(struct ltdb_private *ltdb, struct ldb_val key, struct ldb_val data, int flags);
+ int (*delete)(struct ltdb_private *ltdb, struct ldb_val key);
int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
- int (*update_in_iterate)(struct ltdb_private *ltdb, TDB_DATA key,
- TDB_DATA key2, TDB_DATA data, void *ctx);
+ int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
+ struct ldb_val key2, struct ldb_val data, void *ctx);
int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
int (*parser)(TDB_DATA key, TDB_DATA data,
void *private_data),
--
2.7.4
From a396c5fc2baa82ceb10cad919864644c825972f5 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 19 Feb 2018 12:37:20 +1300
Subject: [PATCH 13/49] ldb: Change remaining fetch prototypes to remove
TDB_DATA
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 16 ++++++++--------
lib/ldb/ldb_tdb/ldb_search.c | 28 +++++++++++++++-------------
lib/ldb/ldb_tdb/ldb_tdb.c | 38 +++++++++++++++++++++++++++++++++++---
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++--
4 files changed, 60 insertions(+), 26 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f05b11b..60909f6 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -364,9 +364,9 @@ done:
return ret;
}
/* Handles only a single record */
-static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
- void *private_data),
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
+ void *private_data),
void *ctx)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
@@ -374,7 +374,7 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
MDB_val mdb_data;
MDB_txn *txn = NULL;
MDB_dbi dbi;
- TDB_DATA data;
+ struct ldb_val data;
txn = get_current_txn(lmdb);
if (txn == NULL) {
@@ -388,8 +388,8 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
if (lmdb->error != MDB_SUCCESS) {
@@ -404,8 +404,8 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
}
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- data.dptr = mdb_data.mv_data;
- data.dsize = mdb_data.mv_size;
+ data.data = mdb_data.mv_data;
+ data.length = mdb_data.mv_size;
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 78ef8b0..35583a1 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -180,17 +180,15 @@ struct ltdb_parse_data_unpack_ctx {
unsigned int unpack_flags;
};
-static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
+static int ltdb_parse_data_unpack(struct ldb_val key,
+ struct ldb_val data,
void *private_data)
{
struct ltdb_parse_data_unpack_ctx *ctx = private_data;
unsigned int nb_elements_in_db;
int ret;
struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
- struct ldb_val data_parse = {
- .data = data.dptr,
- .length = data.dsize
- };
+ struct ldb_val data_parse = data;
if (ctx->unpack_flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
/*
@@ -200,13 +198,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
* and the caller needs a stable result.
*/
data_parse.data = talloc_memdup(ctx->msg,
- data.dptr,
- data.dsize);
+ data.data,
+ data.length);
if (data_parse.data == NULL) {
ldb_debug(ldb, LDB_DEBUG_ERROR,
"Unable to allocate data(%d) for %*.*s\n",
- (int)data.dsize,
- (int)key.dsize, (int)key.dsize, key.dptr);
+ (int)data.length,
+ (int)key.length, (int)key.length, key.data);
return LDB_ERR_OPERATIONS_ERROR;
}
}
@@ -217,13 +215,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
ctx->unpack_flags,
&nb_elements_in_db);
if (ret == -1) {
- if (data_parse.data != data.dptr) {
+ if (data_parse.data != data.data) {
talloc_free(data_parse.data);
}
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %*.*s\n",
- (int)key.dsize, (int)key.dsize, key.dptr);
- return LDB_ERR_OPERATIONS_ERROR;
+ (int)key.length, (int)key.length, key.data);
+ return LDB_ERR_OPERATIONS_ERROR;
}
return ret;
}
@@ -246,13 +244,17 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
.module = module,
.unpack_flags = unpack_flags
};
+ struct ldb_val ldb_key = {
+ .data = tdb_key.dptr,
+ .length = tdb_key.dsize
+ };
memset(msg, 0, sizeof(*msg));
msg->num_elements = 0;
msg->elements = NULL;
- ret = ltdb->kv_ops->fetch_and_parse(ltdb, tdb_key,
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, ldb_key,
ltdb_parse_data_unpack, &ctx);
if (ret == -1) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1e8bd29..038b83e 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1735,6 +1735,9 @@ struct kv_ctx {
ldb_kv_traverse_fn kv_traverse_fn;
void *ctx;
struct ltdb_private *ltdb;
+ int (*parser)(struct ldb_val key,
+ struct ldb_val data,
+ void *private_data);
};
static int ldb_tdb_traverse_fn_wrapper(struct tdb_context *tdb, TDB_DATA tdb_key, TDB_DATA tdb_data, void *ctx)
@@ -1817,12 +1820,41 @@ static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
return tdb_ret;
}
-static int ltdb_tdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
+static int ltdb_tdb_parse_record_wrapper(TDB_DATA tdb_key, TDB_DATA tdb_data,
+ void *ctx)
+{
+ struct kv_ctx *kv_ctx = ctx;
+ struct ldb_val key = {
+ .length = tdb_key.dsize,
+ .data = tdb_key.dptr,
+ };
+ struct ldb_val data = {
+ .length = tdb_data.dsize,
+ .data = tdb_data.dptr,
+ };
+
+ return kv_ctx->parser(key, data, kv_ctx->ctx);
+}
+
+static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
+ struct ldb_val ldb_key,
+ int (*parser)(struct ldb_val key,
+ struct ldb_val data,
void *private_data),
void *ctx)
{
- return tdb_parse_record(ltdb->tdb, key, parser, ctx);
+ struct kv_ctx kv_ctx = {
+ .parser = parser,
+ .ctx = ctx,
+ .ltdb = ltdb
+ };
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+
+ return tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
+ &kv_ctx);
}
static const char * ltdb_tdb_name(struct ltdb_private *ltdb)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 92bbd0e..11396ee 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -15,8 +15,8 @@ struct kv_db_ops {
int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
struct ldb_val key2, struct ldb_val data, void *ctx);
- int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
+ int (*fetch_and_parse)(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
void *private_data),
void *ctx);
int (*lock_read)(struct ldb_module *);
--
2.7.4
From 371867b097f54025f655b7483bcd1bc29fbea82c Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 14/49] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb
This is used in selftest with 'ldb:nosync = true'.
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 60909f6..07c2339 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -661,6 +661,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
if (flags & LDB_FLG_RDONLY) {
mdb_flags |= MDB_RDONLY;
}
+ if (flags & LDB_FLG_NOSYNC) {
+ mdb_flags |= MDB_NOSYNC;
+ }
ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
--
2.7.4
From bac0a05fc7f1ef31044190526a07943e64785e45 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 17 Jan 2018 14:02:09 +1300
Subject: [PATCH 15/49] upgradeprovision: Do not copy backup lmdb -lock files
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
source4/scripting/bin/samba_upgradeprovision | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/source4/scripting/bin/samba_upgradeprovision b/source4/scripting/bin/samba_upgradeprovision
index f3e690b..e0db12e 100755
--- a/source4/scripting/bin/samba_upgradeprovision
+++ b/source4/scripting/bin/samba_upgradeprovision
@@ -1397,9 +1397,10 @@ def backup_provision(paths, dir, only_db):
else:
os.mkdir(os.path.join(dir, "sam.ldb.d"), 0700)
- for ldb in os.listdir(samldbdir):
- tdb_util.tdb_copy(os.path.join(samldbdir, ldb),
- os.path.join(dir, "sam.ldb.d", ldb))
+ for ldb_name in os.listdir(samldbdir):
+ if not ldb_name.endswith("-lock"):
+ tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
+ os.path.join(dir, "sam.ldb.d", ldb_name))
def sync_calculated_attributes(samdb, names):
--
2.7.4
From bc494326eeb9407ca68fa55feaac5e5af81a155e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 16/49] tests: Add tests to check for max key length and DB
size
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_size_test.c | 213 ++++++++++++++++++++++++++++++++
lib/ldb/tests/ldb_lmdb_test.c | 243 +++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 9 ++
3 files changed, 465 insertions(+)
create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
create mode 100644 lib/ldb/tests/ldb_lmdb_test.c
diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 0000000..506c586
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,213 @@
+/*
+ * lmdb backend specific tests for ldb
+ * Tests for truncated index keys
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+ int ret, x;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ const int MB = 1024 * 1024;
+ char *blob = NULL;
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ blob = talloc_zero_size(tmp_ctx, (MB + 1));
+ assert_non_null(blob);
+ memset(blob, 'x', MB);
+
+
+ for (x = 0; x < 6144; x++) {
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+ assert_non_null(msg->dn);
+
+ ldb_transaction_start(test_ctx->ldb);
+ ret = ldb_msg_add_string(msg, "blob", blob);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+ ldb_transaction_commit(test_ctx->ldb);
+
+ TALLOC_FREE(msg);
+ }
+ talloc_free(tmp_ctx);
+ {
+ struct stat s;
+ ret = stat(test_ctx->dbfile, &s);
+ assert_int_equal(ret, 0);
+ assert_true(s.st_size > (6144LL * MB));
+ }
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_db_size_gt_4GB,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 0000000..2902eaf
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,243 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_eq_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 4c32e95..5f9445e 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -434,6 +434,14 @@ def build(bld):
cflags='-DTEST_BE=\"mdb\"',
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_lmdb_test',
+ source='tests/ldb_lmdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+ bld.SAMBA_BINARY('ldb_lmdb_size_test',
+ source='tests/ldb_lmdb_size_test.c',
+ deps='cmocka ldb',
+ install=False)
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
@@ -470,6 +478,7 @@ def test(ctx):
test_exes = ['ldb_tdb_mod_op_test', 'ldb_msg_test']
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
+ test_exes.append('ldb_lmdb_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.7.4
From 51e67e56faf46ab3fa0a665b8bfa457481a9c4ea Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 17/49] ldb_mdb: Store pid to change destructor on fork
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 2 ++
2 files changed, 24 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 07c2339..24ba561 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -587,6 +587,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
{
struct lmdb_trans *ltx = NULL;
+ /* Check if this is a forked child */
+ if (getpid() != lmdb->pid) {
+ int fd = 0;
+ /*
+ * We cannot call mdb_env_close or commit any transactions,
+ * otherwise they might appear finished in the parent.
+ *
+ */
+
+ if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+ close(fd);
+ }
+
+ /* Remove the pointer, so that no access should occur */
+ lmdb->env = NULL;
+
+ return 0;
+ }
+
/*
* Close the read transaction if it's open
*/
@@ -673,6 +692,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
return ldb_mdb_err_map(ret);
}
+ /* Store the original pid during the LMDB open */
+ lmdb->pid = getpid();
+
return LDB_SUCCESS;
}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index e62e353..7e0729c 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -43,6 +43,8 @@ struct lmdb_private {
int error;
MDB_txn *read_txn;
+ pid_t pid;
+
};
struct lmdb_trans {
--
2.7.4
From fb02c2874bcb08aac69939508c0dcc0f40ff5b8f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 18/49] lib ldb tests: Run api and index test on tdb and lmdb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 228 +++++++++++++++++++++++++++++++-----------
lib/ldb/tests/python/index.py | 40 +++++++-
2 files changed, 210 insertions(+), 58 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 85fe1bc..60678c8 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -12,6 +12,9 @@ import shutil
PY3 = sys.version_info > (3, 0)
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
def tempdir():
import tempfile
@@ -44,13 +47,36 @@ class NoContextTests(TestCase):
encoded2 = ldb.binary_encode('test\\x')
self.assertEqual(encoded2, encoded)
-class SimpleLdb(TestCase):
+
+class LdbBaseTest(TestCase):
+ def setUp(self):
+ super(LdbBaseTest, self).setUp()
+ try:
+ if self.prefix is None:
+ self.prefix = TDB_PREFIX
+ except AttributeError:
+ self.prefix = TDB_PREFIX
+
+ def tearDown(self):
+ super(LdbBaseTest, self).tearDown()
+
+ def url(self):
+ return self.prefix + self.filename
+
+ def flags(self):
+ if self.prefix == MDB_PREFIX:
+ return ldb.FLG_NOSYNC
+ else:
+ return 0
+
+
+class SimpleLdb(LdbBaseTest):
def setUp(self):
super(SimpleLdb, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
- self.ldb = ldb.Ldb(self.filename)
+ self.ldb = ldb.Ldb(self.url(), flags=self.flags())
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -58,16 +84,15 @@ class SimpleLdb(TestCase):
# Ensure the LDB is closed now, so we close the FD
del(self.ldb)
-
def test_connect(self):
- ldb.Ldb(self.filename)
+ ldb.Ldb(self.url(), flags=self.flags())
def test_connect_none(self):
ldb.Ldb()
def test_connect_later(self):
x = ldb.Ldb()
- x.connect(self.filename)
+ x.connect(self.url(), flags=self.flags())
def test_repr(self):
x = ldb.Ldb()
@@ -82,7 +107,7 @@ class SimpleLdb(TestCase):
self.assertEqual([], x.modules())
def test_modules_tdb(self):
- x = ldb.Ldb(self.filename)
+ x = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual("[<ldb module 'tdb'>]", repr(x.modules()))
def test_firstmodule_none(self):
@@ -90,53 +115,53 @@ class SimpleLdb(TestCase):
self.assertEqual(x.firstmodule, None)
def test_firstmodule_tdb(self):
- x = ldb.Ldb(self.filename)
+ x = ldb.Ldb(self.url(), flags=self.flags())
mod = x.firstmodule
self.assertEqual(repr(mod), "<ldb module 'tdb'>")
def test_search(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search()), 0)
def test_search_controls(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(controls=["paged_results:0:5"])), 0)
def test_search_attrs(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, ""), ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
def test_search_string_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search("", ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
def test_search_attr_string(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(TypeError, l.search, attrs="dc")
self.assertRaises(TypeError, l.search, attrs=b"dc")
def test_opaque(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.set_opaque("my_opaque", l)
self.assertTrue(l.get_opaque("my_opaque") is not None)
self.assertEqual(None, l.get_opaque("unknown"))
def test_search_scope_base_empty_db(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
ldb.SCOPE_BASE)), 0)
def test_search_scope_onelevel_empty_db(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
ldb.SCOPE_ONELEVEL)), 0)
def test_delete(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError, lambda: l.delete(ldb.Dn(l, "dc=foo2")))
def test_delete_w_unhandled_ctrl(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo1")
m["b"] = [b"a"]
@@ -145,10 +170,10 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_contains(self):
- name = self.filename
- l = ldb.Ldb(name)
+ name = self.url()
+ l = ldb.Ldb(name, flags=self.flags())
self.assertFalse(ldb.Dn(l, "dc=foo3") in l)
- l = ldb.Ldb(name)
+ l = ldb.Ldb(name, flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo3")
m["b"] = ["a"]
@@ -160,23 +185,23 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_get_config_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_config_basedn())
def test_get_root_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_root_basedn())
def test_get_schema_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_schema_basedn())
def test_get_default_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_default_basedn())
def test_add(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
@@ -188,7 +213,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo4"))
def test_search_iterator(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
s = l.search_iterator()
s.abandon()
try:
@@ -288,7 +313,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = "bla"
@@ -300,7 +325,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo4"))
def test_add_w_unhandled_ctrl(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
@@ -308,7 +333,7 @@ class SimpleLdb(TestCase):
self.assertRaises(ldb.LdbError, lambda: l.add(m,["search_options:1:2"]))
def test_add_dict(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
"bla": b"bla"}
self.assertEqual(len(l.search()), 0)
@@ -319,7 +344,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_dict_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
"bla": "bla"}
self.assertEqual(len(l.search()), 0)
@@ -330,7 +355,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_dict_string_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": "dc=foo6", "bla": b"bla"}
self.assertEqual(len(l.search()), 0)
l.add(m)
@@ -340,7 +365,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo6"))
def test_add_dict_bytes_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": b"dc=foo6", "bla": b"bla"}
self.assertEqual(len(l.search()), 0)
l.add(m)
@@ -350,7 +375,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo6"))
def test_rename(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo7")
m["bla"] = b"bla"
@@ -363,7 +388,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=bar"))
def test_rename_string_dns(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo8")
m["bla"] = b"bla"
@@ -377,7 +402,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=bar"))
def test_empty_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(0, len(l.search()))
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=empty")
@@ -394,7 +419,7 @@ class SimpleLdb(TestCase):
self.assertEqual(0, len(rm[0]))
def test_modify_delete(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m["bla"] = [b"1234"]
@@ -417,7 +442,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modifydelete"))
def test_modify_delete_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m.text["bla"] = ["1234"]
@@ -440,7 +465,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modifydelete"))
def test_modify_add(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
@@ -458,7 +483,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_add_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
@@ -476,7 +501,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_replace(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m["bla"] = [b"1234", b"456"]
@@ -496,7 +521,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modify2"))
def test_modify_replace_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m.text["bla"] = ["1234", "456"]
@@ -516,7 +541,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modify2"))
def test_modify_flags_change(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
@@ -542,7 +567,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_flags_change_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
@@ -568,7 +593,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_transaction_commit(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo9"))
m["foo"] = [b"bar"]
@@ -577,7 +602,7 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_transaction_cancel(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo10"))
m["foo"] = [b"bar"]
@@ -588,12 +613,12 @@ class SimpleLdb(TestCase):
def test_set_debug(self):
def my_report_fn(level, text):
pass
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.set_debug(my_report_fn)
def test_zero_byte_string(self):
"""Testing we do not get trapped in the \0 byte in a property string."""
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.add({
"dn" : b"dc=somedn",
"objectclass" : b"user",
@@ -605,10 +630,20 @@ class SimpleLdb(TestCase):
self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
def test_no_crash_broken_expr(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
-class SearchTests(TestCase):
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SimpleLdbLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SimpleLdbLmdb, self).tearDown()
+
+class SearchTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
super(SearchTests, self).tearDown()
@@ -621,7 +656,9 @@ class SearchTests(TestCase):
super(SearchTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "search_test.ldb")
- self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l = ldb.Ldb(self.url(),
+ flags=self.flags(),
+ options=["modules:rdn_name"])
self.l.add({"dn": "@ATTRIBUTES",
"DC": "CASE_INSENSITIVE"})
@@ -1030,6 +1067,16 @@ class SearchTests(TestCase):
self.assertEqual(len(res11), 1)
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SearchTestsLmdb, self).tearDown()
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
@@ -1091,6 +1138,7 @@ class GUIDIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+
class GUIDIndexedDNFilterSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1125,8 +1173,37 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
-class AddModifyTests(TestCase):
+class AddModifyTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
super(AddModifyTests, self).tearDown()
@@ -1138,7 +1215,9 @@ class AddModifyTests(TestCase):
super(AddModifyTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "add_test.ldb")
- self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l = ldb.Ldb(self.url(),
+ flags=self.flags(),
+ options=["modules:rdn_name"])
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
@@ -1266,6 +1345,16 @@ class AddModifyTests(TestCase):
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde3"})
+
+class AddModifyTestsLmdb(AddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(AddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(AddModifyTestsLmdb, self).tearDown()
+
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1378,6 +1467,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
super(TransIndexedAddModifyTests, self).tearDown()
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
class DnTests(TestCase):
@@ -1989,13 +2095,13 @@ class ModuleTests(TestCase):
l = ldb.Ldb(self.filename)
self.assertEqual(["init"], ops)
-class LdbResultTests(TestCase):
+class LdbResultTests(LdbBaseTest):
def setUp(self):
super(LdbResultTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
- self.l = ldb.Ldb(self.filename)
+ self.l = ldb.Ldb(self.url(), flags=self.flags())
self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
@@ -2103,7 +2209,7 @@ class LdbResultTests(TestCase):
del(self.l)
gc.collect()
- child_ldb = ldb.Ldb(self.filename)
+ child_ldb = ldb.Ldb(self.url(), flags=self.flags())
# start a transaction
child_ldb.transaction_start()
@@ -2174,7 +2280,7 @@ class LdbResultTests(TestCase):
del(self.l)
gc.collect()
- child_ldb = ldb.Ldb(self.filename)
+ child_ldb = ldb.Ldb(self.url(), flags=self.flags())
# start a transaction
child_ldb.transaction_start()
@@ -2240,6 +2346,16 @@ class LdbResultTests(TestCase):
self.assertEqual(got_pid, pid)
+class LdbResultTestsLmdb(LdbResultTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(LdbResultTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(LdbResultTestsLmdb, self).tearDown()
+
+
class BadTypeTests(TestCase):
def test_control(self):
l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 239b2bf..72c5c43 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -32,6 +32,9 @@ import shutil
PY3 = sys.version_info > (3, 0)
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
def tempdir():
import tempfile
@@ -52,7 +55,29 @@ def contains(result, dn):
return False
-class MaxIndexKeyLengthTests(TestCase):
+class LdbBaseTest(TestCase):
+ def setUp(self):
+ super(LdbBaseTest, self).setUp()
+ try:
+ if self.prefix is None:
+ self.prefix = TDB_PREFIX
+ except AttributeError:
+ self.prefix = TDB_PREFIX
+
+ def tearDown(self):
+ super(LdbBaseTest, self).tearDown()
+
+ def url(self):
+ return self.prefix + self.filename
+
+ def flags(self):
+ if self.prefix == MDB_PREFIX:
+ return ldb.FLG_NOSYNC
+ else:
+ return 0
+
+
+class MaxIndexKeyLengthTests(LdbBaseTest):
def checkGuids(self, key, guids):
#
# This check relies on the current implementation where the indexes
@@ -89,7 +114,7 @@ class MaxIndexKeyLengthTests(TestCase):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "key_len_test.ldb")
# Note that the maximum key length is set to 50
- self.l = ldb.Ldb(self.filename,
+ self.l = ldb.Ldb(self.url(),
options=[
"modules:rdn_name",
"max_key_len_for_self_test:50"])
@@ -1217,6 +1242,17 @@ class MaxIndexKeyLengthTests(TestCase):
code = e.args[0]
self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
if __name__ == '__main__':
import unittest
unittest.TestProgram()
--
2.7.4
From ae60c6c65337af5a265b1bd10a83e39f74257832 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 19/49] ldb_mdb: Apply LMDB key length restrictions at
key-value layer
We need to enforce the GUID index mode so end-users do not get a supprise
in mid-operation and we enforce a max key length of 511 so that the
index key trunctation is done correctly.
Otherwise the DB will appear to work until a very long key (DN or index)
is used, after which it will be sad.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 +++++++++++++++++++++-
lib/ldb/ldb_tdb/ldb_tdb.c | 17 ++++++++++++++++-
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 24ba561..1aaf6d3 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -30,6 +30,8 @@
#define MDB_URL_PREFIX "mdb://"
#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
#define MEGABYTE (1024*1024)
#define GIGABYTE (1024*1024*1024)
@@ -641,7 +643,8 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
{
int ret;
unsigned int mdb_flags;
-
+ int lmdb_max_key_length;
+
if (flags & LDB_FLG_DONT_CREATE_DB) {
struct stat st;
if (stat(path, &st) != 0) {
@@ -695,6 +698,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
+ lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+ /* This will never happen, but if it does make sure to freak out */
+ if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+ talloc_free(lmdb);
+ return ldb_operr(ldb);
+ }
+
return LDB_SUCCESS;
}
@@ -746,6 +757,15 @@ int lmdb_connect(struct ldb_context *ldb,
if (flags & LDB_FLG_RDONLY) {
ltdb->read_only = true;
}
+
+ /*
+ * This maximum length becomes encoded in the index values so
+ * must never change even if LMDB starts to allow longer keys.
+ * The override option is max_key_len_for_self_test, and is
+ * used for testing only.
+ */
+ ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 038b83e..7f4afb9 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -643,6 +643,15 @@ static int ltdb_add(struct ltdb_context *ctx)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret = LDB_SUCCESS;
+ if (ltdb->max_key_length != 0 &&
+ ltdb->cache->GUID_index_attribute == NULL &&
+ !ldb_dn_is_special(req->op.add.message->dn)) {
+ ldb_set_errstring(ldb_module_get_ctx(module),
+ "Must operate ldb_mdb in GUID "
+ "index mode, but " LTDB_IDXGUID " not set.");
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ret = ltdb_check_special_dn(module, req->op.add.message);
if (ret != LDB_SUCCESS) {
return ret;
@@ -2106,7 +2115,13 @@ int init_store(struct ltdb_private *ltdb,
*_module = module;
/*
- * Set the maximum key length
+ * Set or override the maximum key length
+ *
+ * The ldb_mdb code will have set this to 511, but our tests
+ * set this even smaller (to make the tests more practical).
+ *
+ * This must only be used for the selftest as the length
+ * becomes encoded in the index keys.
*/
{
const char *len_str =
--
2.7.4
From 4c6535c1fee80e03474b7524569d878c4daa69e7 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:10 +1300
Subject: [PATCH 20/49] ldb: Allow tests to operate on ldb_mdb after new
restrictions
The tests were working fine on ldb_mdb but only in the mode without
the GUID index. This mode is not OK for production so has been banned
but this means we need to rework the tests to set an objectGUID on
each record.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++---------------
1 file changed, 121 insertions(+), 64 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 60678c8..2e8d64f 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -77,6 +77,10 @@ class SimpleLdb(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+ try:
+ self.ldb.add(self.index)
+ except AttributeError:
+ pass
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -165,6 +169,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo1")
m["b"] = [b"a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
self.assertRaises(ldb.LdbError, lambda: l.delete(m.dn, ["search_options:1:2"]))
l.delete(m.dn)
@@ -177,6 +182,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo3")
m["b"] = ["a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
self.assertTrue(ldb.Dn(l, "dc=foo3") in l)
@@ -205,6 +211,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -245,6 +252,7 @@ class SimpleLdb(LdbBaseTest):
m1 = ldb.Message()
m1.dn = ldb.Dn(l, "dc=foo4")
m1["bla"] = b"bla"
+ m1["objectUUID"] = b"0123456789abcdef"
l.add(m1)
try:
s = l.search_iterator()
@@ -261,6 +269,7 @@ class SimpleLdb(LdbBaseTest):
m2 = ldb.Message()
m2.dn = ldb.Dn(l, "dc=foo5")
m2["bla"] = b"bla"
+ m2["objectUUID"] = b"0123456789abcdee"
l.add(m2)
s = l.search_iterator()
@@ -317,6 +326,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = "bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -335,7 +345,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": b"bla"}
+ "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -346,7 +357,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_text(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": "bla"}
+ "bla": "bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -356,7 +368,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_string_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": "dc=foo6", "bla": b"bla"}
+ m = {"dn": "dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -366,7 +379,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_bytes_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": b"dc=foo6", "bla": b"bla"}
+ m = {"dn": b"dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -379,6 +393,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo7")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -392,6 +407,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo8")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
self.assertEqual(len(l.search()), 1)
@@ -406,14 +422,17 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(0, len(l.search()))
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=empty")
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search()
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["blah"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -423,6 +442,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual([b"1234"], list(rm["bla"]))
@@ -434,7 +454,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -446,6 +467,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual(["1234"], list(rm.text["bla"]))
@@ -457,7 +479,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -469,6 +492,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -477,7 +501,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -487,6 +511,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -495,7 +520,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -505,6 +530,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m["bla"] = [b"1234", b"456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -513,7 +539,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"789"], list(rm["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -525,6 +551,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m.text["bla"] = ["1234", "456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -533,7 +560,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["789"], list(rm.text["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -545,6 +572,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -553,7 +581,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -571,6 +599,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -579,7 +608,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -597,6 +626,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo9"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
l.transaction_commit()
l.delete(m.dn)
@@ -606,6 +636,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo10"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdee"
l.add(m)
l.transaction_cancel()
self.assertEqual(0, len(l.search(ldb.Dn(l, "dc=foo10"))))
@@ -625,6 +656,7 @@ class SimpleLdb(LdbBaseTest):
"cN" : b"LDAPtestUSER",
"givenname" : b"ldap",
"displayname" : b"foo\0bar",
+ "objectUUID" : b"0123456789abcdef"
})
res = l.search(expression="(dn=dc=somedn)")
self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
@@ -638,6 +670,10 @@ class SimpleLdbLmdb(SimpleLdb):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SimpleLdbLmdb, self).setUp()
def tearDown(self):
@@ -659,6 +695,10 @@ class SearchTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
self.l.add({"dn": "@ATTRIBUTES",
"DC": "CASE_INSENSITIVE"})
@@ -670,7 +710,7 @@ class SearchTests(LdbBaseTest):
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
- "objectUUID": b"0123456789abcddf"})
+ "objectUUID": b"0123456789abcdef"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
"name": b"Admins",
"x": "z", "y": "a",
@@ -1072,6 +1112,9 @@ class SearchTestsLmdb(SearchTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SearchTestsLmdb, self).setUp()
def tearDown(self):
@@ -1129,12 +1172,12 @@ class GUIDIndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedSearchTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDXGUID = True
self.IDXONE = True
@@ -1143,15 +1186,14 @@ class GUIDIndexedDNFilterSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedDNFilterSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
@@ -1159,16 +1201,14 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
"""Test searches using the index including @IDXONE, to ensure
the index doesn't break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDAndOneLevelIndexedSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
self.IDXONE = True
@@ -1218,6 +1258,11 @@ class AddModifyTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
@@ -1346,23 +1391,15 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
-class AddModifyTestsLmdb(AddModifyTests):
-
- def setUp(self):
- self.prefix = MDB_PREFIX
- super(AddModifyTestsLmdb, self).setUp()
-
- def tearDown(self):
- super(AddModifyTestsLmdb, self).tearDown()
-
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ if not hasattr(self, 'index'):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
+ "@IDXONE": [b"1"]}
super(IndexedAddModifyTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
- "@IDXONE": [b"1"]})
def test_duplicate_GUID(self):
try:
@@ -1436,14 +1473,12 @@ class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedAddModifyTests, self).setUp()
- indexlist = {"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
- m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
- self.l.modify(m)
class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
@@ -2102,19 +2137,36 @@ class LdbResultTests(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.l = ldb.Ldb(self.url(), flags=self.flags())
- self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
- self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
- self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
- self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1"})
- self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2"})
- self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3"})
- self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4"})
- self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5"})
- self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6"})
- self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7"})
- self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8"})
- self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9"})
- self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10"})
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+ self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org",
+ "objectUUID": b"0123456789abcde0"})
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users",
+ "objectUUID": b"0123456789abcde2"})
+ self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1",
+ "objectUUID": b"0123456789abcde3"})
+ self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2",
+ "objectUUID": b"0123456789abcde4"})
+ self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3",
+ "objectUUID": b"0123456789abcde5"})
+ self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4",
+ "objectUUID": b"0123456789abcde6"})
+ self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5",
+ "objectUUID": b"0123456789abcde7"})
+ self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6",
+ "objectUUID": b"0123456789abcde8"})
+ self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7",
+ "objectUUID": b"0123456789abcde9"})
+ self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8",
+ "objectUUID": b"0123456789abcdea"})
+ self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9",
+ "objectUUID": b"0123456789abcdeb"})
+ self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10",
+ "objectUUID": b"0123456789abcdec"})
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -2215,7 +2267,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2286,7 +2339,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2350,6 +2404,9 @@ class LdbResultTestsLmdb(LdbResultTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(LdbResultTestsLmdb, self).setUp()
def tearDown(self):
--
2.7.4
From 49ee805e8ceca82b7a97d6be34f4e5eec00ad504 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 6 Mar 2018 17:00:07 +1300
Subject: [PATCH 21/49] ldb: Ignore these tests in mdb test mode
These are tests are specifically for when the GUID index is not in use
which is always in with ldb_mdb.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index b911302..6a7e818 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3389,6 +3389,11 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
char *debug_string = NULL;
char *p = NULL;
+ /* The non-GUID mode is not compatible with mdb */
+ if (strcmp(TEST_BE, "mdb") == 0) {
+ return;
+ }
+
ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
tmp_ctx = talloc_new(test_ctx);
assert_non_null(tmp_ctx);
@@ -3438,6 +3443,11 @@ static void test_ldb_duplicate_dn_logging(void **state)
TALLOC_CTX *tmp_ctx;
char *debug_string = NULL;
+ /* The non-GUID mode is not compatible with mdb */
+ if (strcmp(TEST_BE, "mdb") == 0) {
+ return;
+ }
+
ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
tmp_ctx = talloc_new(test_ctx);
assert_non_null(tmp_ctx);
@@ -3836,6 +3846,7 @@ int main(int argc, const char **argv)
test_ldb_add_to_index_unique_values_required,
ldb_non_unique_index_test_setup,
ldb_non_unique_index_test_teardown),
+ /* These tests are not compatible with mdb */
cmocka_unit_test_setup_teardown(
test_ldb_unique_index_duplicate_logging,
ldb_unique_index_test_setup,
--
2.7.4
From e95161a7f4d3a6b8770d84b9fda8d180d6f01231 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Wed, 7 Mar 2018 15:52:16 +1300
Subject: [PATCH 22/49] ldb_mdb tests: extend ldb_mdb tests for key length
Add extra test cases to improve test coverage of the maximum key length
validation.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 166 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 163 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index 2902eaf..fd82867 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -135,12 +135,22 @@ static int ldbtest_setup(void **state)
{
struct ldbtest_ctx *test_ctx;
int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
ldbtest_noconn_setup((void **) &test_ctx);
ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
*state = test_ctx;
return 0;
}
@@ -170,7 +180,8 @@ static void test_ldb_add_key_len_gt_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -184,8 +195,11 @@ static void test_ldb_add_key_len_gt_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
- assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+ assert_int_equal(ret, LDB_SUCCESS);
talloc_free(tmp_ctx);
}
@@ -207,7 +221,8 @@ static void test_ldb_add_key_len_eq_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -220,12 +235,145 @@ static void test_ldb_add_key_len_eq_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
talloc_free(tmp_ctx);
}
+static int ldbtest_setup_noguid(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static void test_ldb_add_special_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_special_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_dn_no_guid_mode(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
+
+ talloc_free(tmp_ctx);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -237,6 +385,18 @@ int main(int argc, const char **argv)
test_ldb_add_key_len_gt_max,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_eq_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_gt_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_dn_no_guid_mode,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.7.4
From 5bf693eff2ae2e70c8641289827dfa2089571894 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 23 Mar 2018 11:10:25 +1300
Subject: [PATCH 23/49] ldb: Allow GUID index mode to be tested on TDB and
require it for LMDB
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 149 ++++++++++++++++++++++++++++++++++------
lib/ldb/wscript | 12 +++-
2 files changed, 139 insertions(+), 22 deletions(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 6a7e818..74c882a 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -202,6 +202,16 @@ static void test_ldif_message_redacted(void **state)
static int ldbtest_setup(void **state)
{
struct ldbtest_ctx *test_ctx;
+ struct ldb_ldif *ldif;
+#ifdef GUID_IDX
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
+#else
+ const char *index_ldif = "\n";
+#endif
int ret;
ldbtest_noconn_setup((void **) &test_ctx);
@@ -209,6 +219,10 @@ static int ldbtest_setup(void **state)
ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
*state = test_ctx;
return 0;
}
@@ -241,6 +255,9 @@ static void test_ldb_add(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
@@ -279,6 +296,9 @@ static void test_ldb_search(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val1");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcde1");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
@@ -294,6 +314,9 @@ static void test_ldb_search(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val2");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcde2");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
@@ -390,7 +413,8 @@ static void assert_dn_doesnt_exist(struct ldbtest_ctx *test_ctx,
static void add_dn_with_cn(struct ldbtest_ctx *test_ctx,
struct ldb_dn *dn,
- const char *cn_value)
+ const char *cn_value,
+ const char *uuid_value)
{
int ret;
TALLOC_CTX *tmp_ctx;
@@ -409,6 +433,9 @@ static void add_dn_with_cn(struct ldbtest_ctx *test_ctx,
ret = ldb_msg_add_string(msg, "cn", cn_value);
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg, "objectUUID", uuid_value);
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, LDB_SUCCESS);
@@ -428,7 +455,9 @@ static void test_ldb_del(void **state)
dn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "%s", basedn);
assert_non_null(dn);
- add_dn_with_cn(test_ctx, dn, "test_del_cn_val");
+ add_dn_with_cn(test_ctx, dn,
+ "test_del_cn_val",
+ "0123456789abcdef");
ret = ldb_delete(test_ctx->ldb, dn);
assert_int_equal(ret, LDB_SUCCESS);
@@ -552,7 +581,8 @@ static void test_ldb_build_search_req(void **state)
static void add_keyval(struct ldbtest_ctx *test_ctx,
const char *key,
- const char *val)
+ const char *val,
+ const char *uuid)
{
int ret;
struct ldb_message *msg;
@@ -566,6 +596,9 @@ static void add_keyval(struct ldbtest_ctx *test_ctx,
ret = ldb_msg_add_string(msg, key, val);
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", uuid);
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
@@ -601,7 +634,8 @@ static void test_transactions(void **state)
ret = ldb_transaction_start(test_ctx->ldb);
assert_int_equal(ret, 0);
- add_keyval(test_ctx, "vegetable", "carrot");
+ add_keyval(test_ctx, "vegetable", "carrot",
+ "0123456789abcde0");
/* commit lev-0 transaction */
ret = ldb_transaction_commit(test_ctx->ldb);
@@ -611,7 +645,8 @@ static void test_transactions(void **state)
ret = ldb_transaction_start(test_ctx->ldb);
assert_int_equal(ret, 0);
- add_keyval(test_ctx, "fruit", "apple");
+ add_keyval(test_ctx, "fruit", "apple",
+ "0123456789abcde1");
/* abort lev-1 nested transaction */
ret = ldb_transaction_cancel(test_ctx->ldb);
@@ -637,14 +672,16 @@ static void test_nested_transactions(void **state)
ret = ldb_transaction_start(test_ctx->ldb);
assert_int_equal(ret, 0);
- add_keyval(test_ctx, "vegetable", "carrot");
+ add_keyval(test_ctx, "vegetable", "carrot",
+ "0123456789abcde0");
/* start another lev-1 nested transaction */
ret = ldb_transaction_start(test_ctx->ldb);
assert_int_equal(ret, 0);
- add_keyval(test_ctx, "fruit", "apple");
+ add_keyval(test_ctx, "fruit", "apple",
+ "0123456789abcde1");
/* abort lev-1 nested transaction */
ret = ldb_transaction_cancel(test_ctx->ldb);
@@ -825,6 +862,7 @@ static int ldb_modify_test_setup(void **state)
struct ldb_mod_test_ctx *mod_test_ctx;
struct keyval kvs[] = {
{ "cn", "test_mod_cn" },
+ { "objectUUID", "0123456789abcdef"},
{ NULL, NULL },
};
@@ -1129,6 +1167,7 @@ static int ldb_search_test_setup(void **state)
{ "cn", "test_search_cn2" },
{ "uid", "test_search_uid" },
{ "uid", "test_search_uid2" },
+ { "objectUUID", "0123456789abcde0"},
{ NULL, NULL },
};
struct keyval kvs2[] = {
@@ -1136,6 +1175,7 @@ static int ldb_search_test_setup(void **state)
{ "cn", "test_search_2_cn2" },
{ "uid", "test_search_2_uid" },
{ "uid", "test_search_2_uid2" },
+ { "objectUUID", "0123456789abcde1"},
{ NULL, NULL },
};
@@ -1532,6 +1572,12 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
exit(LDB_ERR_OPERATIONS_ERROR);
}
+ ret = ldb_msg_add_string(msg, "objectUUID",
+ "0123456789abcdef");
+ if (ret != 0) {
+ exit(ret);
+ }
+
ret = ldb_add(ctx->test_ctx->ldb, msg);
if (ret != 0) {
exit(ret);
@@ -1894,13 +1940,16 @@ static void test_ldb_modify_during_search(void **state, bool add_index,
ret = ldb_msg_add_string(msg, "@IDXATTR", "cn");
assert_int_equal(ret, LDB_SUCCESS);
-
ret = ldb_add(search_test_ctx->ldb_test_ctx->ldb,
msg);
-
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+ msg->elements[0].flags = LDB_FLAG_MOD_ADD;
+ ret = ldb_modify(search_test_ctx->ldb_test_ctx->ldb,
+ msg);
+ }
assert_int_equal(ret, LDB_SUCCESS);
}
-
+
tevent_loop_allow_nesting(search_test_ctx->ldb_test_ctx->ev);
ctx.basedn
@@ -2436,6 +2485,7 @@ static int ldb_case_test_setup(void **state)
struct keyval kvs[] = {
{ "cn", "CaseInsensitiveValue" },
{ "uid", "CaseSensitiveValue" },
+ { "objectUUID", "0123456789abcdef" },
{ NULL, NULL },
};
@@ -2649,6 +2699,11 @@ static void test_ldb_attrs_index_handler(void **state)
/* Add the index (actually any modify will do) */
while ((ldif = ldb_ldif_read_string(ldb_test_ctx->ldb, &index_ldif))) {
ret = ldb_add(ldb_test_ctx->ldb, ldif->msg);
+ if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+ ldif->msg->elements[0].flags = LDB_FLAG_MOD_ADD;
+ ret = ldb_modify(ldb_test_ctx->ldb,
+ ldif->msg);
+ }
assert_int_equal(ret, LDB_SUCCESS);
}
@@ -2734,7 +2789,8 @@ static int ldb_rename_test_setup(void **state)
add_dn_with_cn(ldb_test_ctx,
rename_test_ctx->basedn,
- "test_rename_cn_val");
+ "test_rename_cn_val",
+ "0123456789abcde0");
*state = rename_test_ctx;
return 0;
@@ -2839,7 +2895,8 @@ static void test_ldb_rename_to_exists(void **state)
add_dn_with_cn(rename_test_ctx->ldb_test_ctx,
new_dn,
- "test_rename_cn_val");
+ "test_rename_cn_val",
+ "0123456789abcde1");
ret = ldb_rename(rename_test_ctx->ldb_test_ctx->ldb,
rename_test_ctx->basedn,
@@ -3006,6 +3063,10 @@ static void test_read_only(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID",
+ "0123456789abcde1");
+ assert_int_equal(ret, LDB_SUCCESS);
+
ret = ldb_add(ro_ldb, msg);
assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
TALLOC_FREE(msg);
@@ -3025,6 +3086,10 @@ static void test_read_only(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID",
+ "0123456789abcde2");
+ assert_int_equal(ret, LDB_SUCCESS);
+
ret = ldb_add(rw_ldb, msg);
assert_int_equal(ret, LDB_SUCCESS);
TALLOC_FREE(msg);
@@ -3098,6 +3163,10 @@ static int ldb_unique_index_test_setup(void **state)
const char *index_ldif = \
"dn: @INDEXLIST\n"
"@IDXATTR: cn\n"
+#ifdef GUID_IDX
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+#endif
"\n";
const char *options[] = {"modules:unique_index_test"};
@@ -3186,6 +3255,10 @@ static void test_ldb_add_unique_value_to_unique_index(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg, "objectUUID",
+ "0123456789abcde1");
+ assert_int_equal(ret, LDB_SUCCESS);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3200,6 +3273,10 @@ static int ldb_non_unique_index_test_setup(void **state)
const char *index_ldif = \
"dn: @INDEXLIST\n"
"@IDXATTR: cn\n"
+#ifdef GUID_IDX
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+#endif
"\n";
const char *options[] = {"modules:unique_index_test"};
@@ -3270,6 +3347,10 @@ static void test_ldb_add_duplicate_value_to_unique_index(void **state)
ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg01, "objectUUID",
+ "0123456789abcde1");
+ assert_int_equal(ret, LDB_SUCCESS);
+
ret = ldb_add(test_ctx->ldb, msg01);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3281,6 +3362,10 @@ static void test_ldb_add_duplicate_value_to_unique_index(void **state)
ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+
+ ret = ldb_msg_add_string(msg02, "objectUUID",
+ "0123456789abcde2");
+ assert_int_equal(ret, LDB_SUCCESS);
ret = ldb_add(test_ctx->ldb, msg02);
assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
@@ -3311,6 +3396,9 @@ static void test_ldb_add_to_index_duplicates_allowed(void **state)
ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg01, "objectUUID",
+ "0123456789abcde1");
+
ret = ldb_add(test_ctx->ldb, msg01);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3323,6 +3411,9 @@ static void test_ldb_add_to_index_duplicates_allowed(void **state)
ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg02, "objectUUID",
+ "0123456789abcde2");
+
ret = ldb_add(test_ctx->ldb, msg02);
assert_int_equal(ret, LDB_SUCCESS);
talloc_free(tmp_ctx);
@@ -3352,6 +3443,9 @@ static void test_ldb_add_to_index_unique_values_required(void **state)
ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg01, "objectUUID",
+ "0123456789abcde1");
+
ret = ldb_add(test_ctx->ldb, msg01);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3364,6 +3458,9 @@ static void test_ldb_add_to_index_unique_values_required(void **state)
ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg02, "objectUUID",
+ "0123456789abcde2");
+
ret = ldb_add(test_ctx->ldb, msg02);
assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
talloc_free(tmp_ctx);
@@ -3389,10 +3486,10 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
char *debug_string = NULL;
char *p = NULL;
- /* The non-GUID mode is not compatible with mdb */
- if (strcmp(TEST_BE, "mdb") == 0) {
- return;
- }
+ /* The GUID mode is not compatible with this test */
+#ifdef GUID_IDX
+ return;
+#endif
ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
tmp_ctx = talloc_new(test_ctx);
@@ -3407,6 +3504,9 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
ret = ldb_msg_add_string(msg01, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg01, "objectUUID",
+ "0123456789abcde1");
+
ret = ldb_add(test_ctx->ldb, msg01);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3419,6 +3519,9 @@ static void test_ldb_unique_index_duplicate_logging(void **state)
ret = ldb_msg_add_string(msg02, "cn", "test_unique_index");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg02, "objectUUID",
+ "0123456789abcde2");
+
ret = ldb_add(test_ctx->ldb, msg02);
assert_int_equal(ret, LDB_ERR_CONSTRAINT_VIOLATION);
@@ -3443,10 +3546,10 @@ static void test_ldb_duplicate_dn_logging(void **state)
TALLOC_CTX *tmp_ctx;
char *debug_string = NULL;
- /* The non-GUID mode is not compatible with mdb */
- if (strcmp(TEST_BE, "mdb") == 0) {
- return;
- }
+ /* The GUID mode is not compatible with this test */
+#ifdef GUID_IDX
+ return;
+#endif
ldb_set_debug(test_ctx->ldb, ldb_debug_string, &debug_string);
tmp_ctx = talloc_new(test_ctx);
@@ -3461,6 +3564,9 @@ static void test_ldb_duplicate_dn_logging(void **state)
ret = ldb_msg_add_string(msg01, "cn", "test_unique_index01");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg01, "objectUUID",
+ "0123456789abcde1");
+
ret = ldb_add(test_ctx->ldb, msg01);
assert_int_equal(ret, LDB_SUCCESS);
@@ -3473,6 +3579,9 @@ static void test_ldb_duplicate_dn_logging(void **state)
ret = ldb_msg_add_string(msg02, "cn", "test_unique_index02");
assert_int_equal(ret, LDB_SUCCESS);
+ ret = ldb_msg_add_string(msg02, "objectUUID",
+ "0123456789abcde2");
+
ret = ldb_add(test_ctx->ldb, msg02);
assert_int_equal(ret, LDB_ERR_ENTRY_ALREADY_EXISTS);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 5f9445e..64dac09 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -428,10 +428,16 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_tdb_guid_mod_op_test',
+ source='tests/ldb_mod_op_test.c',
+ cflags='-DTEST_BE=\"tdb\" -DGUID_IDX=1',
+ deps='cmocka ldb',
+ install=False)
+
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
- cflags='-DTEST_BE=\"mdb\"',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
deps='cmocka ldb',
install=False)
bld.SAMBA_BINARY('ldb_lmdb_test',
@@ -475,7 +481,9 @@ def test(ctx):
print("Python testsuite returned %d" % pyret)
cmocka_ret = 0
- test_exes = ['ldb_tdb_mod_op_test', 'ldb_msg_test']
+ test_exes = ['ldb_tdb_mod_op_test',
+ 'ldb_tdb_guid_mod_op_test',
+ 'ldb_msg_test']
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
test_exes.append('ldb_lmdb_test')
--
2.7.4
From 41f2e993e29ad7b12a413d910efa18ad88f34843 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 24/49] ldb_mdb: Wrap mdb_env_open
Wrap mdb_env_open to ensure that we only have one MDB_env opened per
database in each process
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 155 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 121 insertions(+), 34 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 1aaf6d3..62f7289 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -157,6 +157,9 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -212,6 +215,10 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -628,73 +635,154 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
trans_finished(lmdb, ltx);
ltx = lmdb_private_trans_head(lmdb);
}
-
- mdb_env_close(lmdb->env);
lmdb->env = NULL;
return 0;
}
-static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
- struct ldb_context *ldb,
- const char *path,
- unsigned int flags,
- struct lmdb_private *lmdb)
+struct mdb_env_wrap {
+ struct mdb_env_wrap *next, *prev;
+ dev_t device;
+ ino_t inode;
+ MDB_env *env;
+ int pid;
+};
+
+static struct mdb_env_wrap *mdb_list;
+
+/* destroy the last connection to an mdb */
+static int mdb_env_wrap_destructor(struct mdb_env_wrap *w)
{
+ mdb_env_close(w->env);
+ DLIST_REMOVE(mdb_list, w);
+ return 0;
+}
+
+static int lmdb_open_env(TALLOC_CTX *mem_ctx,
+ MDB_env **env,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags) {
int ret;
- unsigned int mdb_flags;
- int lmdb_max_key_length;
-
- if (flags & LDB_FLG_DONT_CREATE_DB) {
- struct stat st;
- if (stat(path, &st) != 0) {
- return LDB_ERR_UNAVAILABLE;
+ unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ /*
+ * MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+
+ struct mdb_env_wrap *w;
+ struct stat st;
+
+ if (stat(path, &st) == 0) {
+ for (w=mdb_list;w;w=w->next) {
+ if (st.st_dev == w->device && st.st_ino == w->inode) {
+ /*
+ * We must have only one MDB_env per process
+ */
+ if (!talloc_reference(mem_ctx, w)) {
+ return ldb_oom(ldb);
+ }
+ *env = w->env;
+ return LDB_SUCCESS;
+ }
}
}
- ret = mdb_env_create(&lmdb->env);
+ w = talloc(mem_ctx, struct mdb_env_wrap);
+ if (w == NULL) {
+ return ldb_oom(ldb);
+ }
+
+ ret = mdb_env_create(env);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
- /* Close when lmdb is released */
- talloc_set_destructor(lmdb, lmdb_pvt_destructor);
-
- ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ /*
+ * Currently we set a 16Gb maximum database size
+ */
+ ret = mdb_env_set_mapsize(*env, 16LL * GIGABYTE);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not open MDB environment %s: %s\n",
path,
mdb_strerror(ret));
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
- mdb_env_set_maxreaders(lmdb->env, 100000);
- /* MDB_NOSUBDIR implies there is a separate file called path and a
- * separate lockfile called path-lock
+ mdb_env_set_maxreaders(*env, 100000);
+ /*
+ * As we ensure that there is only one MDB_env open per database per
+ * process. We can not use the MDB_RDONLY flag, as another ldb may be
+ * opened in read write mode
*/
- mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
- if (flags & LDB_FLG_RDONLY) {
- mdb_flags |= MDB_RDONLY;
- }
if (flags & LDB_FLG_NOSYNC) {
mdb_flags |= MDB_NOSYNC;
}
- ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+ ret = mdb_env_open(*env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
- talloc_free(lmdb);
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
+ if (stat(path, &st) != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not stat %s:\n",
+ path);
+ TALLOC_FREE(w);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ w->env = *env;
+ w->device = st.st_dev;
+ w->inode = st.st_ino;
+
+ talloc_set_destructor(w, mdb_env_wrap_destructor);
+
+ DLIST_ADD(mdb_list, w);
+
+ return LDB_SUCCESS;
+
+}
+
+static int lmdb_pvt_open(struct lmdb_private *lmdb,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags)
+{
+ int ret;
+ int lmdb_max_key_length;
+
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
+ }
+
+ ret = lmdb_open_env(lmdb, &lmdb->env, ldb, path, flags);
+ if (ret != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
@@ -702,7 +790,6 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* This will never happen, but if it does make sure to freak out */
if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
- talloc_free(lmdb);
return ldb_operr(ldb);
}
@@ -739,17 +826,17 @@ int lmdb_connect(struct ldb_context *ldb,
return LDB_ERR_OPERATIONS_ERROR;
}
- lmdb = talloc_zero(ldb, struct lmdb_private);
+ lmdb = talloc_zero(ltdb, struct lmdb_private);
if (lmdb == NULL) {
TALLOC_FREE(ltdb);
- ldb_oom(ldb);
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_oom(ldb);
}
lmdb->ldb = ldb;
ltdb->kv_ops = &lmdb_key_value_ops;
- ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ ret = lmdb_pvt_open(lmdb, ldb, path, flags);
if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(ltdb);
return ret;
}
--
2.7.4
From ff3ea632cf32d613d362772bae80d72a3d2c2555 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 8 Mar 2018 16:47:59 +1300
Subject: [PATCH 25/49] ldb_mdb: Tests for wrap open
Tests to ensure that the mdb_env wrapping code correctly handles
multiple ldb's point to the same physical database file.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 234 ++++++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 5 +-
2 files changed, 237 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 74c882a..8a31924 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -38,6 +38,12 @@
#define TEST_BE DEFAULT_BE
#endif /* TEST_BE */
+#ifdef TEST_LMDB
+#include "lmdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+#endif
+
struct ldbtest_ctx {
struct tevent_context *ev;
struct ldb_context *ldb;
@@ -3812,6 +3818,224 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
}
}
+#ifdef TEST_LMDB
+static int test_ldb_multiple_connections_callback(struct ldb_request *req,
+ struct ldb_reply *ares)
+{
+ int ret;
+ int pipes[2];
+ char buf[2];
+ int pid, child_pid;
+ int wstatus;
+
+ switch (ares->type) {
+ case LDB_REPLY_ENTRY:
+ break;
+
+ case LDB_REPLY_REFERRAL:
+ return LDB_SUCCESS;
+
+ case LDB_REPLY_DONE:
+ return ldb_request_done(req, LDB_SUCCESS);
+ }
+
+ {
+ /*
+ * We open a new ldb on an ldb that is already open and
+ * then close it.
+ *
+ * If the multiple connection wrapping is correct the
+ * underlying MDB_env will be left open and we should see
+ * an active reader in the child we fork next
+ */
+ struct ldb_context *ldb = NULL;
+ struct tevent_context *ev = NULL;
+ TALLOC_CTX *mem_ctx = talloc_new(NULL);
+
+ ev = tevent_context_init(mem_ctx);
+ assert_non_null(ev);
+
+ ldb = ldb_init(mem_ctx, ev);
+ assert_non_null(ldb);
+
+ ret = ldb_connect(ldb, TEST_BE"://apitest.ldb" , 0, NULL);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+ TALLOC_FREE(ldb);
+ TALLOC_FREE(mem_ctx);
+ }
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct MDB_env *env = NULL;
+ struct MDB_envinfo stat;
+
+ /*
+ * Check that there are exactly two readers on the MDB file
+ * backing the ldb.
+ *
+ */
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_create returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_open(env,
+ "apitest.ldb",
+ MDB_NOSUBDIR | MDB_NOTLS,
+ 0644);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_open returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_info(env, &stat);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_info returned (%d)",
+ ret);
+ exit(ret);
+ }
+ if (stat.me_numreaders != 2) {
+ print_error(__location__
+ " Incorrect number of readers (%d)",
+ stat.me_numreaders);
+ exit(LDB_ERR_CONSTRAINT_VIOLATION);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+ return LDB_SUCCESS;
+
+}
+
+static void test_ldb_close_with_multiple_connections(void **state)
+{
+ struct search_test_ctx *search_test_ctx = NULL;
+ struct ldb_dn *search_dn = NULL;
+ struct ldb_request *req = NULL;
+ int ret = 0;
+
+ search_test_ctx = talloc_get_type_abort(*state, struct search_test_ctx);
+ assert_non_null(search_test_ctx);
+
+ search_dn = ldb_dn_new_fmt(search_test_ctx,
+ search_test_ctx->ldb_test_ctx->ldb,
+ "cn=test_search_cn,"
+ "dc=search_test_entry");
+ assert_non_null(search_dn);
+
+ /*
+ * The search just needs to call DONE, we don't care about the
+ * contents of the search for this test
+ */
+ ret = ldb_build_search_req(&req,
+ search_test_ctx->ldb_test_ctx->ldb,
+ search_test_ctx,
+ search_dn,
+ LDB_SCOPE_SUBTREE,
+ "(&(!(filterAttr=*))"
+ "(cn=test_search_cn))",
+ NULL,
+ NULL,
+ NULL,
+ test_ldb_multiple_connections_callback,
+ NULL);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_request(search_test_ctx->ldb_test_ctx->ldb, req);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+ assert_int_equal(ret, 0);
+}
+
+static struct MDB_env* get_mdb_env(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct MDB_env *env = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ lmdb = ltdb->lmdb_private;
+ assert_non_null(lmdb);
+
+ env = lmdb->env;
+ assert_non_null(env);
+
+ return env;
+}
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ struct MDB_env *env3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+ env3 = get_mdb_env(ldb3);
+
+ assert_ptr_equal(env1, env2);
+ assert_ptr_equal(env1, env3);
+
+}
+#endif
int main(int argc, const char **argv)
{
@@ -3976,6 +4200,16 @@ int main(int argc, const char **argv)
test_ldb_talloc_destructor_transaction_cleanup,
ldbtest_setup,
ldbtest_teardown),
+#ifdef TEST_LMDB
+ cmocka_unit_test_setup_teardown(
+ test_ldb_close_with_multiple_connections,
+ ldb_search_test_setup,
+ ldb_search_test_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
+#endif
};
return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 64dac09..24e5fa5 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -437,8 +437,9 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
- cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
- deps='cmocka ldb',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 ' +
+ '-DTEST_LMDB=1',
+ deps='cmocka ldb lmdb',
install=False)
bld.SAMBA_BINARY('ldb_lmdb_test',
source='tests/ldb_lmdb_test.c',
--
2.7.4
From cda9bbc116b51bef480a8ba807065048a2e87024 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:10:54 +1300
Subject: [PATCH 26/49] ldb index: Fix truncation key length calculation
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 076db10..ccbc644 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -848,6 +848,18 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
unsigned indx_len = 0;
unsigned frmt_len = 0;
+ if (max_key_length < 4) {
+ ldb_asprintf_errstring(
+ ldb,
+ __location__ ": max_key_length of (%u) < 4",
+ max_key_length);
+ }
+ /*
+ * ltdb_key_dn() makes something 4 bytes longer, it adds a leading
+ * "DN=" and a trailing string terninator
+ */
+ max_key_length -= 4;
+
if (attr[0] == '@') {
attr_for_dn = attr;
v = *value;
@@ -911,12 +923,13 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
if (should_b64_encode) {
unsigned vstr_len = 0;
char *vstr = ldb_base64_encode(ldb, (char *)v.data, v.length);
+ unsigned num_separators = 3;
if (!vstr) {
talloc_free(attr_folded);
return NULL;
}
vstr_len = strlen(vstr);
- key_len = 3 + indx_len + attr_len + vstr_len;
+ key_len = num_separators + indx_len + attr_len + vstr_len;
if (key_len > max_key_length) {
unsigned excess = key_len - max_key_length;
frmt_len = vstr_len - excess;
@@ -943,7 +956,8 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
}
talloc_free(vstr);
} else {
- key_len = 2 + indx_len + attr_len + (int)v.length;
+ unsigned num_separators = 2;
+ key_len = num_separators + indx_len + attr_len + (int)v.length;
if (key_len > max_key_length) {
unsigned excess = key_len - max_key_length;
frmt_len = v.length - excess;
--
2.7.4
From 1e06c3274cfbba2ea61be2b89cad179a0dabc086 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:14:09 +1300
Subject: [PATCH 27/49] ldb index tests: test large index key value
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 44 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index fd82867..92715fd 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -204,6 +204,46 @@ static void test_ldb_add_key_len_gt_max(void **state)
talloc_free(tmp_ctx);
}
+static void test_ldb_add_key_len_2x_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = 2 * LMDB_MAX_KEY_SIZE;
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
static void test_ldb_add_key_len_eq_max(void **state)
{
int ret;
@@ -386,6 +426,10 @@ int main(int argc, const char **argv)
ldbtest_setup,
ldbtest_teardown),
cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_2x_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
test_ldb_add_special_key_len_eq_max,
ldbtest_setup_noguid,
ldbtest_teardown),
--
2.7.4
From 040b70a393661bb5175858735982e3176170817a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:15:07 +1300
Subject: [PATCH 28/49] ldb index tests: fix tuncated index length calculation
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/index.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 72c5c43..85cbb17 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -113,11 +113,14 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
super(MaxIndexKeyLengthTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "key_len_test.ldb")
- # Note that the maximum key length is set to 50
+ # Note that the maximum key length is set to 54
+ # This accounts for the 4 bytes added by the dn formatting
+ # a leading dn=, and a trailing zero terminator
+ #
self.l = ldb.Ldb(self.url(),
options=[
"modules:rdn_name",
- "max_key_len_for_self_test:50"])
+ "max_key_len_for_self_test:54"])
self.l.add({"dn": "@ATTRIBUTES",
"uniqueThing": "UNIQUE_INDEX"})
self.l.add({"dn": "@INDEXLIST",
--
2.7.4
From 83837c81a2c2ac687dec4bedb78acc6336274740 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:45:28 +1300
Subject: [PATCH 29/49] ldb index: Add tests for truncated base 64 index keys
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/index.py | 37 ++++++++++++++++++++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 85cbb17..3379fb9 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -124,7 +124,12 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
self.l.add({"dn": "@ATTRIBUTES",
"uniqueThing": "UNIQUE_INDEX"})
self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"uniqueThing", b"notUnique"],
+ "@IDXATTR": [
+ b"uniqueThing",
+ b"notUnique",
+ b"base64____lt",
+ b"base64_____eq",
+ b"base64______gt"],
"@IDXONE": [b"1"],
"@IDXGUID": [b"objectUUID"],
"@IDX_DN_GUID": [b"GUID"]})
@@ -870,6 +875,36 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
contains(res, "OU=12,OU=SEARCH_NON_UNIQUE01,DC=SAMBA,DC=ORG"))
#
+ # Test index key truncation for base64 encoded values
+ #
+ def test_index_truncated_base64_encoded_keys(self):
+ value = b"aaaaaaaaaaaaaaaaaaaa\x02"
+ # base64 encodes to "YWFhYWFhYWFhYWFhYWFhYWFhYWEC"
+
+ # One less than max key length
+ self.l.add({"dn": "OU=01,OU=BASE64,DC=SAMBA,DC=ORG",
+ "base64____lt": value,
+ "objectUUID": b"0123456789abcde0"})
+ self.checkGuids(
+ "@INDEX:BASE64____LT::YWFhYWFhYWFhYWFhYWFhYWFhYWEC",
+ b"0123456789abcde0")
+
+ # Equal max key length
+ self.l.add({"dn": "OU=02,OU=BASE64,DC=SAMBA,DC=ORG",
+ "base64_____eq": value,
+ "objectUUID": b"0123456789abcde1"})
+ self.checkGuids(
+ "@INDEX:BASE64_____EQ::YWFhYWFhYWFhYWFhYWFhYWFhYWEC",
+ b"0123456789abcde1")
+
+ # One greater than max key length
+ self.l.add({"dn": "OU=03,OU=BASE64,DC=SAMBA,DC=ORG",
+ "base64______gt": value,
+ "objectUUID": b"0123456789abcde2"})
+ self.checkGuids(
+ "@INDEX#BASE64______GT##YWFhYWFhYWFhYWFhYWFhYWFhYWE",
+ b"0123456789abcde2")
+ #
# Test adding to non unique index with identical multivalued index
# attributes
#
--
2.7.4
From f245e36fb1d0e2da45ed1779f7b08a9b12270df5 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 12:43:25 +1300
Subject: [PATCH 30/49] ldb test: close pipes to stop forked tests failing on
failure
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 8a31924..108b505 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -1530,6 +1530,7 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
struct ldb_message *msg;
TALLOC_FREE(ctx->test_ctx->ldb);
TALLOC_FREE(ctx->test_ctx->ev);
+ close(pipes[0]);
ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
if (ctx->test_ctx->ev == NULL) {
exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1592,7 +1593,7 @@ static int test_ldb_search_against_transaction_callback1(struct ldb_request *req
ret = ldb_transaction_commit(ctx->test_ctx->ldb);
exit(ret);
}
-
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
@@ -1766,6 +1767,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
struct ldb_dn *dn, *new_dn;
TALLOC_FREE(ctx->test_ctx->ldb);
TALLOC_FREE(ctx->test_ctx->ev);
+ close(pipes[0]);
ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
if (ctx->test_ctx->ev == NULL) {
exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1832,6 +1834,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
struct ldb_message_element *el;
TALLOC_FREE(ctx->test_ctx->ldb);
TALLOC_FREE(ctx->test_ctx->ev);
+ close(pipes[0]);
ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
if (ctx->test_ctx->ev == NULL) {
exit(LDB_ERR_OPERATIONS_ERROR);
@@ -1907,6 +1910,7 @@ static int test_ldb_modify_during_search_callback1(struct ldb_request *req,
* sending the "GO" as it is blocked at ldb_transaction_start().
*/
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
@@ -2081,6 +2085,7 @@ static int test_ldb_modify_during_whole_search_callback1(struct ldb_request *req
struct ldb_message_element *el;
TALLOC_FREE(ctx->test_ctx->ldb);
TALLOC_FREE(ctx->test_ctx->ev);
+ close(pipes[0]);
ctx->test_ctx->ev = tevent_context_init(ctx->test_ctx);
if (ctx->test_ctx->ev == NULL) {
exit(LDB_ERR_OPERATIONS_ERROR);
@@ -2143,6 +2148,7 @@ static int test_ldb_modify_during_whole_search_callback1(struct ldb_request *req
exit(ret);
}
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
@@ -2355,6 +2361,7 @@ static void test_ldb_modify_before_ldb_wait(void **state)
struct ldb_message_element *el;
TALLOC_FREE(search_test_ctx->ldb_test_ctx->ldb);
TALLOC_FREE(search_test_ctx->ldb_test_ctx->ev);
+ close(pipes[0]);
search_test_ctx->ldb_test_ctx->ev = tevent_context_init(search_test_ctx->ldb_test_ctx);
if (search_test_ctx->ldb_test_ctx->ev == NULL) {
exit(LDB_ERR_OPERATIONS_ERROR);
@@ -2423,6 +2430,7 @@ static void test_ldb_modify_before_ldb_wait(void **state)
ret = ldb_transaction_commit(search_test_ctx->ldb_test_ctx->ldb);
exit(ret);
}
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
@@ -3873,6 +3881,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
if (child_pid == 0) {
struct MDB_env *env = NULL;
struct MDB_envinfo stat;
+ close(pipes[0]);
/*
* Check that there are exactly two readers on the MDB file
@@ -3921,6 +3930,7 @@ static int test_ldb_multiple_connections_callback(struct ldb_request *req,
}
exit(LDB_SUCCESS);
}
+ close(pipes[1]);
ret = read(pipes[0], buf, 2);
assert_int_equal(ret, 2);
--
2.7.4
From 51a2d1570c4aaf6d1f00066cab8eb2be58740d08 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 31/49] ldb_mdb: prevent MDB_env reuse across forks
MDB_env's may not be reused accross forks. Check the pid that the lmdb
structure was created by, and return an error if it is being used by a
different process.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 33 +++++-
lib/ldb/tests/ldb_mod_op_test.c | 233 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 264 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 62f7289..7de0a16 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -433,6 +433,18 @@ static int lmdb_lock_read(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct lmdb_private *lmdb = ltdb->lmdb_private;
+ pid_t pid = getpid();
+
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
lmdb->error = MDB_SUCCESS;
if (ltdb->in_transaction == 0 &&
@@ -472,12 +484,25 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
struct lmdb_trans *ltx;
struct lmdb_trans *ltx_head;
MDB_txn *tx_parent;
+ pid_t pid = getpid();
ltx = talloc_zero(lmdb, struct lmdb_trans);
if (ltx == NULL) {
return ldb_oom(lmdb->ldb);
}
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opened by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
+
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
@@ -645,7 +670,7 @@ struct mdb_env_wrap {
dev_t device;
ino_t inode;
MDB_env *env;
- int pid;
+ pid_t pid;
};
static struct mdb_env_wrap *mdb_list;
@@ -672,10 +697,13 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
struct mdb_env_wrap *w;
struct stat st;
+ pid_t pid = getpid();
if (stat(path, &st) == 0) {
for (w=mdb_list;w;w=w->next) {
- if (st.st_dev == w->device && st.st_ino == w->inode) {
+ if (st.st_dev == w->device &&
+ st.st_ino == w->inode &&
+ pid == w->pid) {
/*
* We must have only one MDB_env per process
*/
@@ -746,6 +774,7 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
w->env = *env;
w->device = st.st_dev;
w->inode = st.st_ino;
+ w->pid = pid;
talloc_set_destructor(w, mdb_env_wrap_destructor);
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 108b505..1f5bfd9 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -4045,6 +4045,227 @@ static void test_multiple_opens(void **state)
assert_ptr_equal(env1, env3);
}
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ env3 = get_mdb_env(ldb3);
+ if (env1 != env2) {
+ print_error(__location__": env1 != env2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (env1 == env3) {
+ print_error(__location__": env1 == env3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_transaction_start_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ close(pipes[0]);
+ ret = ldb_transaction_start(ldb1);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_transaction_start "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_lock_read_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_search "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+ {
+ /*
+ * Ensure that the search actually succeeds on the opening
+ * pid
+ */
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ assert_int_equal(0, ret);
+ }
+}
#endif
int main(int argc, const char **argv)
@@ -4219,6 +4440,18 @@ int main(int argc, const char **argv)
test_multiple_opens,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_start_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_lock_read_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
#endif
};
--
2.7.4
From 4e5ee4fa54d08adc0a2a340fb68e0577612bb50d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 16:43:54 +1300
Subject: [PATCH 32/49] ldb: require pid match for cached ldb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 1b12d64..4dc5ca6 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -94,6 +94,8 @@ static struct ldb_wrap {
/* the context is what we use to tell if two ldb
* connections are exactly equivalent
*/
+ pid_t pid; /* We want to re-open in a new PID due to
+ * the LMDB backend */
const char *url;
struct tevent_context *ev;
struct loadparm_context *lp_ctx;
@@ -186,10 +188,12 @@ char *wrap_casefold(void *context, void *mem_ctx, const char *s, size_t n)
struct cli_credentials *credentials,
unsigned int flags)
{
+ pid_t pid = getpid();
struct ldb_wrap *w;
/* see if we can re-use an existing ldb */
for (w=ldb_wrap_list; w; w=w->next) {
- if (w->context.ev == ev &&
+ if (w->context.pid == pid &&
+ w->context.ev == ev &&
w->context.lp_ctx == lp_ctx &&
w->context.session_info == session_info &&
w->context.credentials == credentials &&
@@ -249,6 +253,7 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
return false;
}
+ c.pid = getpid();
c.url = url;
c.ev = ev;
c.lp_ctx = lp_ctx;
--
2.7.4
From 9580f3683012d7be4421b72aca6723c661921db7 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 13:14:38 +1300
Subject: [PATCH 33/49] source3: initilize_password_db after a fork.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
source3/rpc_server/lsasd.c | 3 +++
source3/smbd/process.c | 1 +
source3/smbd/server_exit.c | 6 +++++-
source3/winbindd/winbindd.c | 2 ++
source3/winbindd/winbindd_dual.c | 2 ++
5 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/source3/rpc_server/lsasd.c b/source3/rpc_server/lsasd.c
index 08e9fe2..9dcf462 100644
--- a/source3/rpc_server/lsasd.c
+++ b/source3/rpc_server/lsasd.c
@@ -22,6 +22,7 @@
#include "includes.h"
#include "messages.h"
#include "ntdomain.h"
+#include "passdb.h"
#include "lib/id_cache.h"
@@ -251,6 +252,7 @@ static bool lsasd_child_init(struct tevent_context *ev_ctx,
DEBUG(0,("reinit_after_fork() failed\n"));
smb_panic("reinit_after_fork() failed");
}
+ initialize_password_db(true, ev_ctx);
lsasd_child_id = child_id;
lsasd_reopen_logs(child_id);
@@ -856,6 +858,7 @@ void start_lsasd(struct tevent_context *ev_ctx,
DEBUG(0,("reinit_after_fork() failed\n"));
smb_panic("reinit_after_fork() failed");
}
+ initialize_password_db(true, ev_ctx);
/* save the parent process id so the children can use it later */
parent_id = messaging_server_id(msg_ctx);
diff --git a/source3/smbd/process.c b/source3/smbd/process.c
index 4376f2e..df54a44 100644
--- a/source3/smbd/process.c
+++ b/source3/smbd/process.c
@@ -3412,6 +3412,7 @@ bool fork_echo_handler(struct smbXsrv_connection *xconn)
nt_errstr(status)));
exit(1);
}
+ initialize_password_db(true, xconn->ev_ctx);
smbd_echo_loop(xconn, listener_pipe[1]);
exit(0);
}
diff --git a/source3/smbd/server_exit.c b/source3/smbd/server_exit.c
index dbeb247..50b43a1 100644
--- a/source3/smbd/server_exit.c
+++ b/source3/smbd/server_exit.c
@@ -44,6 +44,7 @@
#include "printing.h"
#include "serverid.h"
#include "messages.h"
+#include "passdb.h"
#include "../lib/util/pidfile.h"
#include "smbprofile.h"
#include "libcli/auth/netlogon_creds_cli.h"
@@ -261,6 +262,9 @@ NTSTATUS smbd_reinit_after_fork(struct messaging_context *msg_ctx,
struct tevent_context *ev_ctx,
bool parent_longlived, const char *comment)
{
+ NTSTATUS ret;
am_parent = NULL;
- return reinit_after_fork(msg_ctx, ev_ctx, parent_longlived, comment);
+ ret = reinit_after_fork(msg_ctx, ev_ctx, parent_longlived, comment);
+ initialize_password_db(true, ev_ctx);
+ return ret;
}
diff --git a/source3/winbindd/winbindd.c b/source3/winbindd/winbindd.c
index 8821f39..b908d91 100644
--- a/source3/winbindd/winbindd.c
+++ b/source3/winbindd/winbindd.c
@@ -44,6 +44,7 @@
#include "lib/async_req/async_sock.h"
#include "libsmb/samlogon_cache.h"
#include "libcli/auth/netlogon_creds_cli.h"
+#include "passdb.h"
#undef DBGC_CLASS
#define DBGC_CLASS DBGC_WINBIND
@@ -1752,6 +1753,7 @@ int main(int argc, const char **argv)
if (!NT_STATUS_IS_OK(status)) {
exit_daemon("Winbindd reinit_after_fork() failed", map_errno_from_nt_status(status));
}
+ initialize_password_db(true, server_event_context());
/*
* Do not initialize the parent-child-pipe before becoming
diff --git a/source3/winbindd/winbindd_dual.c b/source3/winbindd/winbindd_dual.c
index 5ae5bbd..cff2c97 100644
--- a/source3/winbindd/winbindd_dual.c
+++ b/source3/winbindd/winbindd_dual.c
@@ -40,6 +40,7 @@
#include "lib/param/loadparm.h"
#include "lib/util/sys_rw.h"
#include "lib/util/sys_rw_data.h"
+#include "passdb.h"
#undef DBGC_CLASS
#define DBGC_CLASS DBGC_WINBIND
@@ -1503,6 +1504,7 @@ NTSTATUS winbindd_reinit_after_fork(const struct winbindd_child *myself,
DEBUG(0,("reinit_after_fork() failed\n"));
return status;
}
+ initialize_password_db(true, server_event_context());
close_conns_after_fork();
--
2.7.4
From 79ac1f4ccda0e878dbb30db3ed4283a56e41cc55 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:20:35 +1300
Subject: [PATCH 34/49] ldb tests: ldb_mod_op_test use correct ldb to create dn
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 1f5bfd9..fdad330 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3094,7 +3094,7 @@ static void test_read_only(void **state)
msg = ldb_msg_new(tmp_ctx);
assert_non_null(msg);
- msg->dn = ldb_dn_new_fmt(msg, ro_ldb, "dc=test");
+ msg->dn = ldb_dn_new_fmt(msg, rw_ldb, "dc=test");
assert_non_null(msg->dn);
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
--
2.7.4
From 2683f9c0a95fbfdc7726883289e0198067e09da9 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:33:32 +1300
Subject: [PATCH 35/49] ldb_tdb: ltdb_tdb_parse_record map tdb error codes
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 7f4afb9..1103b52 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1861,9 +1861,14 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
.dptr = ldb_key.data,
.dsize = ldb_key.length
};
+ int ret;
- return tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
- &kv_ctx);
+ ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
+ &kv_ctx);
+ if (ret == 0) {
+ return LDB_SUCCESS;
+ }
+ return ltdb_err_map(tdb_error(ltdb->tdb));
}
static const char * ltdb_tdb_name(struct ltdb_private *ltdb)
--
2.7.4
From 6daf51cee29855a75f111929ae86b790e127f3ac Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:36:33 +1300
Subject: [PATCH 36/49] ldb_tdb: ltdb_tdb_store require active transaction
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 1103b52..d72271e 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -424,6 +424,9 @@ static int ltdb_tdb_store(struct ltdb_private *ltdb, struct ldb_val ldb_key,
.dptr = ldb_data.data,
.dsize = ldb_data.length
};
+ if (!tdb_transaction_active(ltdb->tdb)){
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
return tdb_store(ltdb->tdb, key, data, flags);
}
--
2.7.4
From 7bf34346448a7afdb265725d8aac5e9d7d768361 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 15 Mar 2018 11:37:06 +1300
Subject: [PATCH 37/49] ldb_tdb: ltdb_tdb_delete require active transaction
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index d72271e..f4cd197 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -678,6 +678,9 @@ static int ltdb_tdb_delete(struct ltdb_private *ltdb, struct ldb_val ldb_key)
.dptr = ldb_key.data,
.dsize = ldb_key.length
};
+ if (!tdb_transaction_active(ltdb->tdb)){
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
return tdb_delete(ltdb->tdb, tdb_key);
}
--
2.7.4
From 559ffae4d2df4bafbb86210869991f647baed441 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 11:25:28 +1300
Subject: [PATCH 38/49] ldb: make backends expose if there is an active
transaction
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 37 +++++++++++++++++++++----------------
lib/ldb/ldb_tdb/ldb_tdb.c | 5 +++++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
3 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 7de0a16..bc1550a 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -91,6 +91,11 @@ static int lmdb_error_at(struct ldb_context *ldb,
return ldb_err;
}
+
+static bool lmdb_transaction_active(struct ltdb_private *ltdb) {
+ return ltdb->lmdb_private->txlist != NULL;
+}
+
static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
{
if (ltx == NULL) {
@@ -581,23 +586,23 @@ static bool lmdb_changed(struct ltdb_private *ltdb)
return true;
}
-
static struct kv_db_ops lmdb_key_value_ops = {
- .store = lmdb_store,
- .delete = lmdb_delete,
- .iterate = lmdb_traverse_fn,
- .update_in_iterate = lmdb_update_in_iterate,
- .fetch_and_parse = lmdb_parse_record,
- .lock_read = lmdb_lock_read,
- .unlock_read = lmdb_unlock_read,
- .begin_write = lmdb_transaction_start,
- .prepare_write = lmdb_transaction_prepare_commit,
- .finish_write = lmdb_transaction_commit,
- .abort_write = lmdb_transaction_cancel,
- .error = lmdb_error,
- .errorstr = lmdb_errorstr,
- .name = lmdb_name,
- .has_changed = lmdb_changed,
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+ .transaction_active = lmdb_transaction_active,
};
static const char *lmdb_get_path(const char *url)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index f4cd197..75779ea 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1892,6 +1892,10 @@ static bool ltdb_tdb_changed(struct ltdb_private *ltdb)
return has_changed;
}
+static bool ltdb_transaction_active(struct ltdb_private *ltdb) {
+ return tdb_transaction_active(ltdb->tdb);
+}
+
static const struct kv_db_ops key_value_ops = {
.store = ltdb_tdb_store,
.delete = ltdb_tdb_delete,
@@ -1908,6 +1912,7 @@ static const struct kv_db_ops key_value_ops = {
.errorstr = ltdb_errorstr,
.name = ltdb_tdb_name,
.has_changed = ltdb_tdb_changed,
+ .transaction_active = ltdb_transaction_active,
};
static void ltdb_callback(struct tevent_context *ev,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 11396ee..b3ba691 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -29,6 +29,7 @@ struct kv_db_ops {
const char * (*errorstr)(struct ltdb_private *ltdb);
const char * (*name)(struct ltdb_private *ltdb);
bool (*has_changed)(struct ltdb_private *ltdb);
+ bool (*transaction_active)(struct ltdb_private *ltdb);
};
/* this private structure is used by the ltdb backend in the
--
2.7.4
From 7aea593106311494c91da0513eaad0f84f7e5f49 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:23:39 +1300
Subject: [PATCH 39/49] ldb: Unwind transaction counter if start_transaction
fails
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/common/ldb.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/common/ldb.c b/lib/ldb/common/ldb.c
index a4d9977..2249089 100644
--- a/lib/ldb/common/ldb.c
+++ b/lib/ldb/common/ldb.c
@@ -379,6 +379,7 @@ int ldb_transaction_start(struct ldb_context *ldb)
"ldb transaction start: %s (%d)",
ldb_strerror(status),
status);
+ ldb->transaction_active--;
}
if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "start ldb transaction error: %s",
--
2.7.4
From 86bb9867f01eeb61fffa6ace7dff1f7655733295 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:24:46 +1300
Subject: [PATCH 40/49] ldb_mdb: Improve error message when no transaction is
active
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index bc1550a..bb398ba 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -392,7 +392,7 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
txn = get_current_txn(lmdb);
if (txn == NULL) {
- ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction active");
lmdb->error = MDB_PANIC;
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
--
2.7.4
From 1f4f1a81bb8883fd8d55568170810461d8a4589e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:27:10 +1300
Subject: [PATCH 41/49] ldb: Do not make search or DB modifications without a
lock
The ldb_cache startup code would previously not take a read lock nor a sufficiently wide write transaction.
The new code takes a read lock, and if it needs to write takes a write lock (transaction)
and re-reads before continuing.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 50 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 38 insertions(+), 12 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 4790bcd..0d16452 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -386,6 +386,7 @@ int ltdb_cache_load(struct ldb_module *module)
uint64_t seq;
struct ldb_message *baseinfo = NULL, *options = NULL;
const struct ldb_schema_attribute *a;
+ bool have_write_txn = false;
int r;
ldb = ldb_module_get_ctx(module);
@@ -406,29 +407,38 @@ int ltdb_cache_load(struct ldb_module *module)
baseinfo_dn = ldb_dn_new(baseinfo, ldb, LTDB_BASEINFO);
if (baseinfo_dn == NULL) goto failed;
+ r = ltdb->kv_ops->lock_read(module);
+ if (r != LDB_SUCCESS) {
+ goto failed;
+ }
r= ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0);
if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
- goto failed;
+ goto failed_and_unlock;
}
-
+
/* possibly initialise the baseinfo */
if (r == LDB_ERR_NO_SUCH_OBJECT) {
+ /* Give up the read lock, try again with a write lock */
+ r = ltdb->kv_ops->unlock_read(module);
+ if (r != LDB_SUCCESS) {
+ goto failed;
+ }
+
if (ltdb->kv_ops->begin_write(ltdb) != 0) {
goto failed;
}
+ have_write_txn = true;
+
/* error handling for ltdb_baseinfo_init() is by
looking for the record again. */
ltdb_baseinfo_init(module);
- if (ltdb->kv_ops->finish_write(ltdb) != 0) {
- goto failed;
- }
-
if (ltdb_search_dn1(module, baseinfo_dn, baseinfo, 0) != LDB_SUCCESS) {
- goto failed;
+ goto failed_and_unlock;
}
+
}
/* Ignore the result, and update the sequence number */
@@ -443,16 +453,17 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb->sequence_number = seq;
/* Read an interpret database options */
+
options = ldb_msg_new(ltdb->cache);
- if (options == NULL) goto failed;
+ if (options == NULL) goto failed_and_unlock;
options_dn = ldb_dn_new(options, ldb, LTDB_OPTIONS);
- if (options_dn == NULL) goto failed;
+ if (options_dn == NULL) goto failed_and_unlock;
r= ltdb_search_dn1(module, options_dn, options, 0);
talloc_free(options_dn);
if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
- goto failed;
+ goto failed_and_unlock;
}
/* set flags if they do exist */
@@ -479,7 +490,7 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb_attributes_unload(module);
if (ltdb_index_load(module, ltdb) == -1) {
- goto failed;
+ goto failed_and_unlock;
}
/*
@@ -488,7 +499,7 @@ int ltdb_cache_load(struct ldb_module *module)
* partition module.
*/
if (ltdb_attributes_load(module) == -1) {
- goto failed;
+ goto failed_and_unlock;
}
ltdb->GUID_index_syntax = NULL;
@@ -503,10 +514,25 @@ int ltdb_cache_load(struct ldb_module *module)
}
done:
+ if (have_write_txn) {
+ if (ltdb->kv_ops->finish_write(ltdb) != 0) {
+ goto failed;
+ }
+ } else {
+ ltdb->kv_ops->unlock_read(module);
+ }
+
talloc_free(options);
talloc_free(baseinfo);
return 0;
+failed_and_unlock:
+ if (have_write_txn) {
+ ltdb->kv_ops->abort_write(ltdb);
+ } else {
+ ltdb->kv_ops->unlock_read(module);
+ }
+
failed:
talloc_free(options);
talloc_free(baseinfo);
--
2.7.4
From 07b029377d9110956ffff7229dd9033a878ed641 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:29:25 +1300
Subject: [PATCH 42/49] ldb: Remove implicit read lock and remove transaction
counter
The way to know if we are in a transaction is if there is a non-NULL transaction handle.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 51 +++++++++++++++--------------------------------
1 file changed, 16 insertions(+), 35 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index bb398ba..3c2dc98 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -129,26 +129,21 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
return ltx;
}
+
static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
{
MDB_txn *txn;
- if (lmdb->read_txn != NULL) {
- return lmdb->read_txn;
- }
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
- if (txn == NULL) {
- int ret;
- ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
- if (ret != 0) {
- lmdb->error = ret;
- ldb_asprintf_errstring(lmdb->ldb,
- "%s failed: %s\n", __FUNCTION__,
- mdb_strerror(ret));
- }
- lmdb->read_txn = txn;
+ if (txn != NULL) {
+ return txn;
+ }
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
}
- return txn;
+ lmdb->error = MDB_BAD_TXN;
+ ldb_set_errstring(lmdb->ldb, __location__":No active transaction\n");
+ return NULL;
}
static int lmdb_store(struct ltdb_private *ltdb,
@@ -193,10 +188,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_val value;
lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
if (lmdb->error != MDB_SUCCESS) {
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
mdb_flags = 0;
@@ -301,11 +292,6 @@ done:
mdb_cursor_close(cursor);
}
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
-
if (lmdb->error != MDB_SUCCESS) {
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
@@ -409,10 +395,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
if (lmdb->error != MDB_SUCCESS) {
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
if (lmdb->error == MDB_NOTFOUND) {
return LDB_ERR_NO_SUCH_OBJECT;
}
@@ -424,11 +406,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- /* We created a read transaction, commit it */
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return parser(key, data, ctx);
}
@@ -452,7 +429,7 @@ static int lmdb_lock_read(struct ldb_module *module)
}
lmdb->error = MDB_SUCCESS;
- if (ltdb->in_transaction == 0 &&
+ if (lmdb_transaction_active(ltdb) == false &&
ltdb->read_lock_count == 0) {
lmdb->error = mdb_txn_begin(lmdb->env,
NULL,
@@ -472,7 +449,7 @@ static int lmdb_unlock_read(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ if (lmdb_transaction_active(ltdb) == false && ltdb->read_lock_count == 1) {
struct lmdb_private *lmdb = ltdb->lmdb_private;
mdb_txn_commit(lmdb->read_txn);
lmdb->read_txn = NULL;
@@ -491,6 +468,11 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
MDB_txn *tx_parent;
pid_t pid = getpid();
+ /* Do not take out the transaction lock on a read-only DB */
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ltx = talloc_zero(lmdb, struct lmdb_trans);
if (ltx == NULL) {
return ldb_oom(lmdb->ldb);
@@ -507,7 +489,6 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
return LDB_ERR_PROTOCOL_ERROR;
}
-
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
--
2.7.4
From 57201731cd943301e8769fd97c9583f6b942c24c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:28:18 +1300
Subject: [PATCH 43/49] ldb_tdb: Disallow TDB nested transactions and use
tdb_transaction_active()
This avoids keeping a counter, which can be error-prone.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 27 +++++++++++++++------------
lib/ldb/ldb_tdb/ldb_tdb.h | 1 -
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 75779ea..4458936 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -101,7 +101,7 @@ static int ltdb_lock_read(struct ldb_module *module)
int tdb_ret = 0;
int ret;
- if (ltdb->in_transaction == 0 &&
+ if (tdb_transaction_active(ltdb->tdb) == false &&
ltdb->read_lock_count == 0) {
tdb_ret = tdb_lockall_read(ltdb->tdb);
}
@@ -128,7 +128,7 @@ static int ltdb_unlock_read(struct ldb_module *module)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
tdb_unlockall_read(ltdb->tdb);
ltdb->read_lock_count--;
return 0;
@@ -379,7 +379,7 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
/* only allow modifies inside a transaction, otherwise the
* ldb is unsafe */
- if (ltdb->in_transaction == 0) {
+ if (ltdb->kv_ops->transaction_active(ltdb) == false) {
ldb_set_errstring(ldb_module_get_ctx(module), "ltdb modify without transaction");
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -1472,7 +1472,6 @@ static int ltdb_start_trans(struct ldb_module *module)
return ltdb->kv_ops->error(ltdb);
}
- ltdb->in_transaction++;
ltdb_index_transaction_start(module);
@@ -1485,20 +1484,21 @@ static int ltdb_prepare_commit(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- if (ltdb->in_transaction != 1) {
- return LDB_SUCCESS;
+ if (!ltdb->kv_ops->transaction_active(ltdb)) {
+ ldb_set_errstring(ldb_module_get_ctx(module),
+ "ltdb_prepare_commit() called "
+ "without transaction active");
+ return LDB_ERR_OPERATIONS_ERROR;
}
ret = ltdb_index_transaction_commit(module);
if (ret != LDB_SUCCESS) {
ltdb->kv_ops->abort_write(ltdb);
- ltdb->in_transaction--;
return ret;
}
if (ltdb->kv_ops->prepare_write(ltdb) != 0) {
ret = ltdb->kv_ops->error(ltdb);
- ltdb->in_transaction--;
ldb_debug_set(ldb_module_get_ctx(module),
LDB_DEBUG_FATAL,
"Failure during "
@@ -1526,7 +1526,6 @@ static int ltdb_end_trans(struct ldb_module *module)
}
}
- ltdb->in_transaction--;
ltdb->prepared_commit = false;
if (ltdb->kv_ops->finish_write(ltdb) != 0) {
@@ -1546,7 +1545,6 @@ static int ltdb_del_trans(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- ltdb->in_transaction--;
if (ltdb_index_transaction_cancel(module) != 0) {
ltdb->kv_ops->abort_write(ltdb);
@@ -1776,7 +1774,7 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
.ctx = ctx,
.ltdb = ltdb
};
- if (ltdb->in_transaction != 0) {
+ if (tdb_transaction_active(ltdb->tdb)) {
return tdb_traverse(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
} else {
return tdb_traverse_read(ltdb->tdb, ldb_tdb_traverse_fn_wrapper, &kv_ctx);
@@ -1869,6 +1867,11 @@ static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
};
int ret;
+ if (tdb_transaction_active(ltdb->tdb) == false &&
+ ltdb->read_lock_count == 0) {
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
ret = tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
&kv_ctx);
if (ret == 0) {
@@ -2181,7 +2184,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
path = url;
}
- tdb_flags = TDB_DEFAULT | TDB_SEQNUM;
+ tdb_flags = TDB_DEFAULT | TDB_SEQNUM | TDB_DISALLOW_NESTING;
/* check for the 'nosync' option */
if (flags & LDB_FLG_NOSYNC) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index b3ba691..4c96038 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -54,7 +54,6 @@ struct ltdb_private {
const char *GUID_index_dn_component;
} *cache;
- int in_transaction;
bool check_base;
bool disallow_dn_filter;
--
2.7.4
From 45228a1af647f490b6fee3d10dc0dcef18943562 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:14:10 +1300
Subject: [PATCH 44/49] ldb tests: api ensure database correctly populated
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 2e8d64f..21d5d8c 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -1316,6 +1316,19 @@ class AddModifyTests(LdbBaseTest):
"name": b"Admins",
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde2"})
+
+ res2 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde1)")
+ self.assertEqual(len(res2), 1)
+ self.assertEqual(str(res2[0].dn), "OU=DUP,DC=SAMBA,DC=ORG")
+
+ res3 = self.l.search(base="DC=SAMBA,DC=ORG",
+ scope=ldb.SCOPE_SUBTREE,
+ expression="(objectUUID=0123456789abcde2)")
+ self.assertEqual(len(res3), 1)
+ self.assertEqual(str(res3[0].dn), "OU=DUP2,DC=SAMBA,DC=ORG")
+
try:
self.l.rename("OU=DUP,DC=SAMBA,DC=ORG",
"OU=DUP2,DC=SAMBA,DC=ORG")
--
2.7.4
From 8760ed3b87f287b3afd77cd3fcc34a98118ece47 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 20 Mar 2018 12:15:12 +1300
Subject: [PATCH 45/49] ldb tests: add cmocka tests of kv operations
Add tests for the behaviour the ldb layer expects the key value layer to
provide. This should make it easier to add another KV store
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_kv_ops_test.c | 1579 +++++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 15 +-
2 files changed, 1593 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/tests/ldb_kv_ops_test.c
diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
new file mode 100644
index 0000000..fda49c7
--- /dev/null
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -0,0 +1,1579 @@
+/*
+ * Tests exercising the ldb key value operations.
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+
+/*
+ * A KV module is expected to have the following behaviour
+ *
+ * - A transaction must be open to perform any read, write or delete operation
+ * - Writes and Deletes should not be visible until a transaction is commited
+ * - Nested transactions are not permitted
+ * - transactions can be rolled back and commited.
+ * - supports iteration over all records in the database
+ * - supports the update_in_iterate operation allowing entries to be
+ * re-keyed.
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+#include "ldb_tdb/ldb_tdb.h"
+
+
+#define DEFAULT_BE "tdb"
+
+#ifndef TEST_BE
+#define TEST_BE DEFAULT_BE
+#endif /* TEST_BE */
+
+#define NUM_RECS 1024
+
+
+struct test_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct test_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int noconn_setup(void **state)
+{
+ struct test_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct test_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "kvopstest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int noconn_teardown(void **state)
+{
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int setup(void **state)
+{
+ struct test_ctx *test_ctx;
+ int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
+
+ noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
+ *state = test_ctx;
+ return 0;
+}
+
+static int teardown(void **state)
+{
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static struct ltdb_private *get_ltdb(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ return ltdb;
+}
+
+static int parse(struct ldb_val key,
+ struct ldb_val data,
+ void *private_data)
+{
+ struct ldb_val* read = private_data;
+
+ /* Yes, we essentially leak this. That is OK */
+ read->data = talloc_size(talloc_autofree_context(),
+ data.length);
+ assert_non_null(read->data);
+
+ memcpy(read->data, data.data, data.length);
+ read->length = data.length;
+ return LDB_SUCCESS;
+}
+
+/*
+ * Test that data can be written to the kv store and be read back.
+ */
+static void test_add_get(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ talloc_free(tmp_ctx);
+}
+/*
+ * Test that attempts to read data without a read transaction fail.
+ */
+static void test_read_outside_transaction(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ * Note there is no read transaction active
+ */
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+/*
+ * Test that data can be deleted from the kv store
+ */
+static void test_delete(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now delete it.
+ */
+ ret = ltdb->kv_ops->delete(ltdb, key);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now try to read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+/*
+ * Check that writes are correctly rolled back when a transaction
+ * is rolled back.
+ */
+static void test_transaction_abort_write(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+
+
+ /*
+ * Now abort the transaction
+ */
+ ret = ltdb->kv_ops->abort_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back, should not be there
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+/*
+ * Check that deletes are correctly rolled back when a transaction is
+ * aborted.
+ */
+static void test_transaction_abort_delete(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now delete it.
+ */
+ ret = ltdb->kv_ops->delete(ltdb, key);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, LDB_ERR_NO_SUCH_OBJECT);
+
+ /*
+ * Abort the transaction
+ */
+ ret = ltdb->kv_ops->abort_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now try to read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+/*
+ * Test that writes outside a transaction fail
+ */
+static void test_write_outside_transaction(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Attempt to write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+/*
+ * Test data can not be deleted outside a transaction
+ */
+static void test_delete_outside_transaction(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ uint8_t key_val[] = "TheKey";
+ struct ldb_val key = {
+ .data = key_val,
+ .length = sizeof(key_val)
+ };
+
+ uint8_t value[] = "The record contents";
+ struct ldb_val data = {
+ .data = value,
+ .length = sizeof(value)
+ };
+
+ struct ldb_val read;
+
+ int flags = 0;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the record
+ */
+ ret = ltdb->kv_ops->store(ltdb, key, data, flags);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now attempt to delete a record
+ */
+ ret = ltdb->kv_ops->delete(ltdb, key);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ /*
+ * And now read it back
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &read);
+ assert_int_equal(ret, 0);
+ assert_int_equal(sizeof(value), read.length);
+ assert_memory_equal(value, read.data, sizeof(value));
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+static int traverse_fn(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val data,
+ void *ctx) {
+
+ int *visits = ctx;
+ int i;
+
+ if (strncmp("key ", (char *) key.data, 4) == 0) {
+ i = strtol((char *) &key.data[4], NULL, 10);
+ visits[i]++;
+ }
+ return LDB_SUCCESS;
+}
+
+
+/*
+ * Test that iterate visits all the records.
+ */
+static void test_iterate(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ int i;
+ int num_recs = 1024;
+ int visits[num_recs];
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the records
+ */
+ for (i = 0; i < num_recs; i++) {
+ struct ldb_val key;
+ struct ldb_val rec;
+ int flags = 0;
+
+ visits[i] = 0;
+ key.data = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+ key.length = strlen((char *)key.data) + 1;
+
+ rec.data = (uint8_t *) talloc_asprintf(tmp_ctx,
+ "data for record (%04d)",
+ i);
+ rec.length = strlen((char *)rec.data) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(key.data);
+ TALLOC_FREE(rec.data);
+ }
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now iterate over the kv store and ensure that all the
+ * records are visited.
+ */
+ ret = ltdb->kv_ops->lock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+ ret = ltdb->kv_ops->iterate(ltdb, traverse_fn, visits);
+ for (i = 0; i <num_recs; i++) {
+ assert_int_equal(1, visits[i]);
+ }
+ ret = ltdb->kv_ops->unlock_read(test_ctx->ldb->modules);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(tmp_ctx);
+}
+
+struct update_context {
+ struct ldb_context* ldb;
+ int visits[NUM_RECS];
+};
+
+static int update_fn(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val data,
+ void *ctx) {
+
+ struct ldb_val new_key;
+ struct ldb_module *module = NULL;
+ struct update_context *context =NULL;
+ int ret = LDB_SUCCESS;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(ltdb);
+ assert_non_null(tmp_ctx);
+
+ context = talloc_get_type_abort(ctx, struct update_context);
+
+ module = talloc_zero(tmp_ctx, struct ldb_module);
+ module->ldb = context->ldb;
+
+ if (strncmp("key ", (char *) key.data, 4) == 0) {
+ int i = strtol((char *) &key.data[4], NULL, 10);
+ context->visits[i]++;
+ new_key.data = talloc_memdup(tmp_ctx, key.data, key.length);
+ new_key.length = key.length;
+ new_key.data[0] = 'K';
+
+ ret = ltdb->kv_ops->update_in_iterate(ltdb,
+ key,
+ new_key,
+ data,
+ &module);
+ }
+ TALLOC_FREE(tmp_ctx);
+ return ret;
+}
+
+/*
+ * Test that update_in_iterate behaves as expected.
+ */
+static void test_update_in_iterate(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ int i;
+ struct update_context *context = NULL;
+
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ context = talloc_zero(tmp_ctx, struct update_context);
+ assert_non_null(context);
+ context->ldb = test_ctx->ldb;
+ /*
+ * Begin a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Write the records
+ */
+ for (i = 0; i < NUM_RECS; i++) {
+ struct ldb_val key;
+ struct ldb_val rec;
+ int flags = 0;
+
+ key.data = (uint8_t *)talloc_asprintf(tmp_ctx, "key %04d", i);
+ key.length = strlen((char *)key.data) + 1;
+
+ rec.data = (uint8_t *) talloc_asprintf(tmp_ctx,
+ "data for record (%04d)",
+ i);
+ rec.length = strlen((char *)rec.data) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, rec, flags);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(key.data);
+ TALLOC_FREE(rec.data);
+ }
+
+ /*
+ * Commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Now iterate over the kv store and ensure that all the
+ * records are visited.
+ */
+
+ /*
+ * Needs to be done inside a transaction
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ ret = ltdb->kv_ops->iterate(ltdb, update_fn, context);
+ for (i = 0; i < NUM_RECS; i++) {
+ assert_int_equal(1, context->visits[i]);
+ }
+
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ TALLOC_FREE(tmp_ctx);
+}
+
+/*
+ * Ensure that writes are not visible until the transaction has been
+ * committed.
+ */
+static void test_write_transaction_isolation(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ struct ldb_val key;
+ struct ldb_val val;
+
+ const char *KEY1 = "KEY01";
+ const char *VAL1 = "VALUE01";
+
+ const char *KEY2 = "KEY02";
+ const char *VAL2 = "VALUE02";
+
+ /*
+ * Pipes etc to co-ordinate the processes
+ */
+ int to_child[2];
+ int to_parent[2];
+ char buf[2];
+ pid_t pid, w_pid;
+ int wstatus;
+
+ TALLOC_CTX *tmp_ctx;
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ /*
+ * Add a record to the database
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL1);
+ val.length = strlen(VAL1) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+ assert_int_equal(ret, 0);
+
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+
+ ret = pipe(to_child);
+ assert_int_equal(ret, 0);
+ ret = pipe(to_parent);
+ assert_int_equal(ret, 0);
+ /*
+ * Now fork a new process
+ */
+
+ pid = fork();
+ if (pid == 0) {
+
+ struct ldb_context *ldb = NULL;
+ close(to_child[1]);
+ close(to_parent[0]);
+
+ /*
+ * Wait for the transaction to start
+ */
+ ret = read(to_child[0], buf, 2);
+ if (ret != 2) {
+ print_error(__location__": read returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ldb = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb, test_ctx->dbpath, 0, NULL);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ltdb = get_ltdb(ldb);
+
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": lock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ /*
+ * Check that KEY1 is there
+ */
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL1) + 1) != val.length) {
+ print_error(__location__": KEY1 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL1) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+ print_error(__location__": KEY1 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL1,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ /*
+ * Check that KEY2 is not there
+ */
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2 + 1);
+
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": lock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_ERR_NO_SUCH_OBJECT) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ /*
+ * Signal the other process to commit the transaction
+ */
+ ret = write(to_parent[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__": write returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Wait for the transaction to be commited
+ */
+ ret = read(to_child[0], buf, 2);
+ if (ret != 2) {
+ print_error(__location__": read returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Check that KEY1 is there
+ */
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL1) + 1) != val.length) {
+ print_error(__location__": KEY1 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL1) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+ print_error(__location__": KEY1 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL1,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+
+ /*
+ * Check that KEY2 is there
+ */
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL2) + 1) != val.length) {
+ print_error(__location__": KEY2 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL2) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL2, val.data, strlen(VAL2)) != 0) {
+ print_error(__location__": KEY2 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL2,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ exit(0);
+ }
+ close(to_child[0]);
+ close(to_parent[1]);
+
+ /*
+ * Begin a transaction and add a record to the database
+ * but leave the transaction open
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2) + 1;
+
+ val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL2);
+ val.length = strlen(VAL2) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+ assert_int_equal(ret, 0);
+
+ /*
+ * Signal the child process
+ */
+ ret = write(to_child[1], "GO", 2);
+ assert_int_equal(2, ret);
+
+ /*
+ * Wait for the child process to check the DB state while the
+ * transaction is active
+ */
+ ret = read(to_parent[0], buf, 2);
+ assert_int_equal(2, ret);
+
+ /*
+ * commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(0, ret);
+
+ /*
+ * Signal the child process
+ */
+ ret = write(to_child[1], "GO", 2);
+ assert_int_equal(2, ret);
+
+ w_pid = waitpid(pid, &wstatus, 0);
+ assert_int_equal(pid, w_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+
+ TALLOC_FREE(tmp_ctx);
+}
+
+/*
+ * Ensure that deletes are not visible until the transaction has been
+ * committed.
+ */
+static void test_delete_transaction_isolation(void **state)
+{
+ int ret;
+ struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct test_ctx);
+ struct ltdb_private *ltdb = get_ltdb(test_ctx->ldb);
+ struct ldb_val key;
+ struct ldb_val val;
+
+ const char *KEY1 = "KEY01";
+ const char *VAL1 = "VALUE01";
+
+ const char *KEY2 = "KEY02";
+ const char *VAL2 = "VALUE02";
+
+ /*
+ * Pipes etc to co-ordinate the processes
+ */
+ int to_child[2];
+ int to_parent[2];
+ char buf[2];
+ pid_t pid, w_pid;
+ int wstatus;
+
+ TALLOC_CTX *tmp_ctx;
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ /*
+ * Add records to the database
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL1);
+ val.length = strlen(VAL1) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+ assert_int_equal(ret, 0);
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2) + 1;
+
+ val.data = (uint8_t *)talloc_strdup(tmp_ctx, VAL2);
+ val.length = strlen(VAL2) + 1;
+
+ ret = ltdb->kv_ops->store(ltdb, key, val, 0);
+ assert_int_equal(ret, 0);
+
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(ret, 0);
+
+
+ ret = pipe(to_child);
+ assert_int_equal(ret, 0);
+ ret = pipe(to_parent);
+ assert_int_equal(ret, 0);
+ /*
+ * Now fork a new process
+ */
+
+ pid = fork();
+ if (pid == 0) {
+
+ struct ldb_context *ldb = NULL;
+ close(to_child[1]);
+ close(to_parent[0]);
+
+ /*
+ * Wait for the transaction to be started
+ */
+ ret = read(to_child[0], buf, 2);
+ if (ret != 2) {
+ print_error(__location__": read returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ ldb = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb, test_ctx->dbpath, 0, NULL);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ltdb = get_ltdb(ldb);
+
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": lock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ /*
+ * Check that KEY1 is there
+ */
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL1) + 1) != val.length) {
+ print_error(__location__": KEY1 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL1) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+ print_error(__location__": KEY1 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL1,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Check that KEY2 is there
+ */
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL2) + 1) != val.length) {
+ print_error(__location__": KEY2 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL2) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL2, val.data, strlen(VAL2)) != 0) {
+ print_error(__location__": KEY2 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL2,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ /*
+ * Signal the other process to commit the transaction
+ */
+ ret = write(to_parent[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__": write returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Wait for the transaction to be commited
+ */
+ ret = read(to_child[0], buf, 2);
+ if (ret != 2) {
+ print_error(__location__": read returned (%d)\n",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Check that KEY1 is there
+ */
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY1);
+ key.length = strlen(KEY1) + 1;
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ if ((strlen(VAL1) + 1) != val.length) {
+ print_error(__location__": KEY1 value lengths different"
+ ", expected (%d) actual(%d)\n",
+ (int)(strlen(VAL1) + 1), (int)val.length);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (memcmp(VAL1, val.data, strlen(VAL1)) != 0) {
+ print_error(__location__": KEY1 values different, "
+ "expected (%s) actual(%s)\n",
+ VAL1,
+ val.data);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+
+ /*
+ * Check that KEY2 is not there
+ */
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2 + 1);
+
+ ret = ltdb->kv_ops->lock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": lock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, key, parse, &val);
+ if (ret != LDB_ERR_NO_SUCH_OBJECT) {
+ print_error(__location__": fetch_and_parse returned "
+ "(%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ ret = ltdb->kv_ops->unlock_read(ldb->modules);
+ if (ret != LDB_SUCCESS) {
+ print_error(__location__": unlock_read returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+
+ exit(0);
+ }
+ close(to_child[0]);
+ close(to_parent[1]);
+
+ /*
+ * Begin a transaction and delete a record from the database
+ * but leave the transaction open
+ */
+ ret = ltdb->kv_ops->begin_write(ltdb);
+ assert_int_equal(ret, 0);
+
+ key.data = (uint8_t *)talloc_strdup(tmp_ctx, KEY2);
+ key.length = strlen(KEY2) + 1;
+
+ ret = ltdb->kv_ops->delete(ltdb, key);
+ assert_int_equal(ret, 0);
+ /*
+ * Signal the child process
+ */
+ ret = write(to_child[1], "GO", 2);
+ assert_int_equal(2, ret);
+
+ /*
+ * Wait for the child process to check the DB state while the
+ * transaction is active
+ */
+ ret = read(to_parent[0], buf, 2);
+ assert_int_equal(2, ret);
+
+ /*
+ * commit the transaction
+ */
+ ret = ltdb->kv_ops->finish_write(ltdb);
+ assert_int_equal(0, ret);
+
+ /*
+ * Signal the child process
+ */
+ ret = write(to_child[1], "GO", 2);
+ assert_int_equal(2, ret);
+
+ w_pid = waitpid(pid, &wstatus, 0);
+ assert_int_equal(pid, w_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+
+ TALLOC_FREE(tmp_ctx);
+}
+
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_add_get,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_delete,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_abort_write,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_abort_delete,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_read_outside_transaction,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_write_outside_transaction,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_delete_outside_transaction,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_iterate,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_update_in_iterate,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_write_transaction_isolation,
+ setup,
+ teardown),
+ cmocka_unit_test_setup_teardown(
+ test_delete_transaction_isolation,
+ setup,
+ teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 24e5fa5..1526ce7 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -434,6 +434,12 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_tdb_kv_ops_test',
+ source='tests/ldb_kv_ops_test.c',
+ cflags='-DTEST_BE=\"tdb\"',
+ deps='cmocka ldb',
+ install=False)
+
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
@@ -449,6 +455,11 @@ def build(bld):
source='tests/ldb_lmdb_size_test.c',
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
+ source='tests/ldb_kv_ops_test.c',
+ cflags='-DTEST_BE=\"mdb\"',
+ deps='cmocka ldb',
+ install=False)
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
@@ -484,10 +495,12 @@ def test(ctx):
cmocka_ret = 0
test_exes = ['ldb_tdb_mod_op_test',
'ldb_tdb_guid_mod_op_test',
- 'ldb_msg_test']
+ 'ldb_msg_test',
+ 'ldb_tdb_kv_ops_test']
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
test_exes.append('ldb_lmdb_test')
+ test_exes.append('ldb_mdb_kv_ops_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.7.4
From 9ca1f402b2f9ccce60040c432679939e3928c4fb Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 8 Mar 2018 14:01:50 +1300
Subject: [PATCH 46/49] ldb: Fix missing NULL terminator in ldb_mod_op_test
testsuite
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index fdad330..da20485 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3182,7 +3182,7 @@ static int ldb_unique_index_test_setup(void **state)
"@IDX_DN_GUID: GUID\n"
#endif
"\n";
- const char *options[] = {"modules:unique_index_test"};
+ const char *options[] = {"modules:unique_index_test", NULL};
ret = ldb_register_module(&ldb_unique_index_test_module_ops);
@@ -3292,7 +3292,7 @@ static int ldb_non_unique_index_test_setup(void **state)
"@IDX_DN_GUID: GUID\n"
#endif
"\n";
- const char *options[] = {"modules:unique_index_test"};
+ const char *options[] = {"modules:unique_index_test", NULL};
ret = ldb_register_module(&ldb_unique_index_test_module_ops);
--
2.7.4
From c0b78c63940096abce1f95be1a325bd6705402d1 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 21 Mar 2018 11:38:22 +1300
Subject: [PATCH 47/49] ldb_mdb: handle EBADE from mdb_env_open
Under some circumstances mdb_env_open returns EBADE, we treat this as
indicating the file is not a valid lmdb format file.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 3c2dc98..f607afe 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -42,6 +42,7 @@ int ldb_mdb_err_map(int lmdb_err)
return LDB_SUCCESS;
case EIO:
return LDB_ERR_OPERATIONS_ERROR;
+ case EBADE:
case MDB_INCOMPATIBLE:
case MDB_CORRUPTED:
case MDB_INVALID:
@@ -792,7 +793,7 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
/* Close when lmdb is released */
--
2.7.4
From 472e7b1efd962b2297b0761985cc09a39c4ef3f0 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 22 Mar 2018 12:50:45 +1300
Subject: [PATCH 48/49] upgradeprovision: detect and handle lmdb databases
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
source4/scripting/bin/samba_upgradeprovision | 26 ++++++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/source4/scripting/bin/samba_upgradeprovision b/source4/scripting/bin/samba_upgradeprovision
index e0db12e..62b7001 100755
--- a/source4/scripting/bin/samba_upgradeprovision
+++ b/source4/scripting/bin/samba_upgradeprovision
@@ -36,11 +36,13 @@ sys.path.insert(0, "bin/python")
import ldb
import samba
import samba.getopt as options
+from samba.samdb import get_default_backend_store
from base64 import b64encode
from samba.credentials import DONT_USE_KERBEROS
from samba.auth import system_session, admin_session
from samba import tdb_util
+from samba import mdb_util
from ldb import (SCOPE_SUBTREE, SCOPE_BASE,
FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE,
MessageElement, Message, Dn, LdbError)
@@ -1366,7 +1368,7 @@ def update_samdb(ref_samdb, samdb, names, provisionUSNs, schema, prereloadfunc):
return 0
-def backup_provision(paths, dir, only_db):
+def backup_provision(samdb, paths, dir, only_db):
"""This function backup the provision files so that a rollback
is possible
@@ -1374,8 +1376,20 @@ def backup_provision(paths, dir, only_db):
:param dir: Directory where to store the backup
:param only_db: Skip sysvol for users with big sysvol
"""
+
+ # Currently we default to tdb for the backend store type
+ #
+ backend_store = "tdb"
+ res = samdb.search(base="@PARTITION",
+ scope=ldb.SCOPE_BASE,
+ attrs=["backendStore"])
+ if "backendStore" in res[0]:
+ backend_store = res[0]["backendStore"][0]
+
+
if paths.sysvol and not only_db:
copytree_with_xattrs(paths.sysvol, os.path.join(dir, "sysvol"))
+
tdb_util.tdb_copy(paths.samdb, os.path.join(dir, os.path.basename(paths.samdb)))
tdb_util.tdb_copy(paths.secrets, os.path.join(dir, os.path.basename(paths.secrets)))
tdb_util.tdb_copy(paths.idmapdb, os.path.join(dir, os.path.basename(paths.idmapdb)))
@@ -1399,8 +1413,12 @@ def backup_provision(paths, dir, only_db):
for ldb_name in os.listdir(samldbdir):
if not ldb_name.endswith("-lock"):
- tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
- os.path.join(dir, "sam.ldb.d", ldb_name))
+ if backend_store == "mdb" and ldb_name != "metadata.tdb":
+ mdb_util.mdb_copy(os.path.join(samldbdir, ldb_name),
+ os.path.join(dir, "sam.ldb.d", ldb_name))
+ else:
+ tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
+ os.path.join(dir, "sam.ldb.d", ldb_name))
def sync_calculated_attributes(samdb, names):
@@ -1571,7 +1589,7 @@ if __name__ == '__main__':
ldbs = get_ldbs(paths, creds, session, lp)
backupdir = tempfile.mkdtemp(dir=paths.private_dir,
prefix="backupprovision")
- backup_provision(paths, backupdir, opts.db_backup_only)
+ backup_provision(ldbs.sam, paths, backupdir, opts.db_backup_only)
try:
ldbs.startTransactions()
--
2.7.4
From 4a5e45f1bf7c8899109f7a06b64c58802fb10cc3 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 20 Mar 2018 12:58:02 +1300
Subject: [PATCH 49/49] ldb: Release ldb 1.4.0
* New LMDB backend
* Comprehensive tests for index behaviour
* Enforce transactions for writes
* Enforce read lock use for all reads
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ABI/ldb-1.4.0.sigs | 279 ++++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.4.0.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 284 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
diff --git a/lib/ldb/ABI/ldb-1.4.0.sigs b/lib/ldb/ABI/ldb-1.4.0.sigs
new file mode 100644
index 0000000..a31b84e
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.4.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.4.0.sigs b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
new file mode 100644
index 0000000..74d6719
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
new file mode 100644
index 0000000..74d6719
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 1526ce7..9ec8c4c 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.3.2'
+VERSION = '1.4.0'
blddir = 'bin'
--
2.7.4
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 473 bytes
Desc: OpenPGP digital signature
URL: <http://lists.samba.org/pipermail/samba-technical/attachments/20180323/de6d2b13/signature-0001.sig>
More information about the samba-technical
mailing list