LMDB key value backend for ldb_tdb (to be renamed ldb_key_val)
Andrew Bartlett
abartlet at samba.org
Tue Mar 6 06:41:43 UTC 2018
On Mon, 2018-03-05 at 18:31 +1300, Andrew Bartlett via samba-technical
wrote:
> On Thu, 2018-02-22 at 11:45 +1300, Garming Sam via samba-technical
> wrote:
> > Hi,
> >
> > This is our current set of patches for implementing an LMDB based
> > backend for LDB. The work is based on a prototype I wrote around this
> > time last year inspired by Jakub's efforts. In saying that, the approach
> > I took was completely different. The idea was to refactor ldb_tdb to be
> > agnostic about which database backend was being used. The advantage has
> > been quite minimal amount of code required to implement a functional
> > 64-bit database backend. Many of the performance optimizations made for
> > ldb_tdb can simply be reused, but conversely, for now we have deferred
> > re-thinking the overall architecture e.g. consolidating the partitions
> > into a single file using LMDB sub databases.
>
> G'Day Garming,
>
> Attached is a few of your patches that I've reviewed, to prepare the
> way for this in Samba master. Thanks for the cross-review in the
> office today.
G'Day Team,
Attached are the current patches to add an lmdb backend, controlled
with an --with-lmdb configure option (currently off) to Samba and ldb.
We are on the last leg of this journey, with just some tests of the
key-value operations to write and a little work on the rest of the code
(fork() handling primarily). We are re-working the existing ldb cmokca
and python tests to operate against LMDB. The patch set is attached
here for comment.
Andrew Bartlett
--
Andrew Bartlett http://samba.org/~abartlet/
Authentication Developer, Samba Team http://samba.org
Samba Developer, Catalyst IT http://catalyst.net.nz/services/samba
-------------- next part --------------
From 57007a74f6b32d52a72dc241f15f11829626a99e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 14:27:40 +1300
Subject: [PATCH 01/20] Change name to sam.ldb to align with new assumptions
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
python/samba/tests/samba3sam.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/samba/tests/samba3sam.py b/python/samba/tests/samba3sam.py
index 929523b768d..bfc7932689f 100644
--- a/python/samba/tests/samba3sam.py
+++ b/python/samba/tests/samba3sam.py
@@ -73,7 +73,7 @@ class MapBaseTestCase(TestCaseInTempDir):
def make_s4dn(basedn, rdn):
return "%s,%s" % (rdn, basedn)
- self.ldbfile = os.path.join(self.tempdir, "test.ldb")
+ self.ldbfile = os.path.join(self.tempdir, "sam.ldb")
self.ldburl = "tdb://" + self.ldbfile
tempdir = self.tempdir
--
2.14.3
From 2a4ee406cd08aa89946534e887ecd1e4ad993dee Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 02/20] ldb_mdb: Implement the lmdb backend for ldb
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 710 ++++++++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 57 ++++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
lib/ldb/tools/ldbdump.c | 123 +++++++-
lib/ldb/wscript | 65 ++++-
wscript | 10 +
6 files changed, 961 insertions(+), 5 deletions(-)
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 00000000000..f4648e9e2ec
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,710 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "lmdb.h"
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX "mdb://"
+#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+
+#define MEGABYTE (1024*1024)
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+ switch (lmdb_err) {
+ case MDB_SUCCESS:
+ return LDB_SUCCESS;
+ case EIO:
+ return LDB_ERR_OPERATIONS_ERROR;
+ case MDB_INCOMPATIBLE:
+ case MDB_CORRUPTED:
+ case MDB_INVALID:
+ return LDB_ERR_UNAVAILABLE;
+ case MDB_BAD_TXN:
+ case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+ case MDB_BAD_DBI:
+#endif
+ case MDB_PANIC:
+ case EINVAL:
+ return LDB_ERR_PROTOCOL_ERROR;
+ case MDB_MAP_FULL:
+ case MDB_DBS_FULL:
+ case MDB_READERS_FULL:
+ case MDB_TLS_FULL:
+ case MDB_TXN_FULL:
+ case EAGAIN:
+ return LDB_ERR_BUSY;
+ case MDB_KEYEXIST:
+ return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ case MDB_NOTFOUND:
+ case ENOENT:
+ return LDB_ERR_NO_SUCH_OBJECT;
+ case EACCES:
+ return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+ default:
+ break;
+ }
+ return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+ int ecode,
+ const char *file,
+ int line)
+{
+ int ldb_err = ldb_mdb_err_map(ecode);
+ char *reason = mdb_strerror(ecode);
+ ldb_asprintf_errstring(ldb,
+ "(%d) - %s at %s:%d",
+ ecode,
+ reason,
+ file,
+ line);
+ return ldb_err;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+ if (ltx == NULL) {
+ return NULL;
+ }
+
+ return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ if (lmdb->txlist) {
+ talloc_steal(lmdb->txlist, ltx);
+ }
+
+ DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ DLIST_REMOVE(lmdb->txlist, ltx);
+ talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx;
+
+ ltx = lmdb->txlist;
+ return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+ MDB_txn *txn;
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
+ }
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ int ret;
+ ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ lmdb->error = ret;
+ ldb_asprintf_errstring(lmdb->ldb,
+ "%s failed: %s\n", __FUNCTION__,
+ mdb_strerror(ret));
+ }
+ lmdb->read_txn = txn;
+ }
+ return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+ TDB_DATA key,
+ TDB_DATA data, int flags)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ int mdb_flags;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ mdb_data.mv_size = data.dsize;
+ mdb_data.mv_data = data.dptr;
+
+ if (flags == TDB_INSERT) {
+ mdb_flags = MDB_NOOVERWRITE;
+ } else if ((flags == TDB_MODIFY)) {
+ /*
+ * Modifying a record, ensure that it exists.
+ * This mimics the TDB semantics
+ */
+ MDB_val value;
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+ if (lmdb->error != MDB_SUCCESS) {
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ mdb_flags = 0;
+ } else {
+ mdb_flags = 0;
+ }
+
+ lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+ ldb_kv_traverse_fn fn,
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+ MDB_cursor *cursor = NULL;
+ int ret;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+ if (lmdb->error != MDB_SUCCESS) {
+ goto done;
+ }
+
+ while ((lmdb->error = mdb_cursor_get(
+ cursor, &mdb_key,
+ &mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+ struct ldb_val key = {
+ .length = mdb_key.mv_size,
+ .data = mdb_key.mv_data,
+ };
+ struct ldb_val data = {
+ .length = mdb_data.mv_size,
+ .data = mdb_data.mv_data,
+ };
+
+ ret = fn(ltdb, key, data, ctx);
+ if (ret != 0) {
+ goto done;
+ }
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ lmdb->error = MDB_SUCCESS;
+ }
+done:
+ if (cursor != NULL) {
+ mdb_cursor_close(cursor);
+ }
+
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+ TDB_DATA key,
+ TDB_DATA key2,
+ TDB_DATA data,
+ void *state)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct TDB_DATA copy;
+ int ret = LDB_SUCCESS;
+
+ /*
+ * Need to take a copy of the data as the delete operation alters the
+ * data, as it is in private lmdb memory.
+ */
+ copy.dsize = data.dsize;
+ copy.dptr = talloc_memdup(ltdb, data.dptr, data.dsize);
+ if (copy.dptr == NULL) {
+ lmdb->error = MDB_PANIC;
+ return ldb_oom(lmdb->ldb);
+ }
+
+ lmdb->error = lmdb_delete(ltdb, key);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to delete %*.*s "
+ "for rekey as %*.*s: %s",
+ (int)key.dsize, (int)key.dsize,
+ (const char *)key.dptr,
+ (int)key2.dsize, (int)key2.dsize,
+ (const char *)key.dptr,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+ lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to rekey %*.*s as %*.*s: %s",
+ (int)key.dsize, (int)key.dsize,
+ (const char *)key.dptr,
+ (int)key2.dsize, (int)key2.dsize,
+ (const char *)key.dptr,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+
+done:
+ if (copy.dptr != NULL) {
+ TALLOC_FREE(copy.dptr);
+ copy.dsize = 0;
+ }
+
+ /*
+ * Explicity invalidate the data, as the delete has done this
+ */
+ data.dsize = 0;
+ data.dptr = NULL;
+
+ return ret;
+}
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
+ int (*parser)(TDB_DATA key, TDB_DATA data,
+ void *private_data),
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ TDB_DATA data;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.dsize;
+ mdb_key.mv_data = key.dptr;
+
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+ if (lmdb->error != MDB_SUCCESS) {
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ data.dptr = mdb_data.mv_data;
+ data.dsize = mdb_data.mv_size;
+
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+
+ /* We created a read transaction, commit it */
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ lmdb->error = MDB_SUCCESS;
+ if (ltdb->in_transaction == 0 &&
+ ltdb->read_lock_count == 0) {
+ lmdb->error = mdb_txn_begin(lmdb->env,
+ NULL,
+ MDB_RDONLY,
+ &lmdb->read_txn);
+ }
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ ltdb->read_lock_count++;
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+ if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+ }
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct lmdb_trans *ltx;
+ struct lmdb_trans *ltx_head;
+ MDB_txn *tx_parent;
+
+ ltx = talloc_zero(lmdb, struct lmdb_trans);
+ if (ltx == NULL) {
+ return ldb_oom(lmdb->ldb);
+ }
+
+ ltx_head = lmdb_private_trans_head(lmdb);
+
+ tx_parent = lmdb_trans_get_tx(ltx_head);
+
+ lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, <x->tx);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ trans_push(lmdb, ltx);
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+ /* No need to prepare a commit */
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ lmdb->error = mdb_txn_commit(ltx->tx);
+ trans_finished(lmdb, ltx);
+
+ return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+ return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+ return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+ return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+ /*
+ * lmdb does no provide a quick way to determine if the database
+ * has changed. This function always returns true.
+ *
+ * Note that tdb uses a sequence number that allows this function
+ * to be implemented efficiently.
+ */
+ return true;
+}
+
+
+static struct kv_db_ops lmdb_key_value_ops = {
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+ const char *path;
+
+ /* parse the url */
+ if (strchr(url, ':')) {
+ if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+ return NULL;
+ }
+ path = url + MDB_URL_PREFIX_SIZE;
+ } else {
+ path = url;
+ }
+
+ return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx = NULL;
+
+ /*
+ * Close the read transaction if it's open
+ */
+ if (lmdb->read_txn != NULL) {
+ mdb_txn_abort(lmdb->read_txn);
+ }
+
+ if (lmdb->env == NULL) {
+ return 0;
+ }
+
+ /*
+ * Abort any currently active transactions
+ */
+ ltx = lmdb_private_trans_head(lmdb);
+ while (ltx != NULL) {
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ ltx = lmdb_private_trans_head(lmdb);
+ }
+
+ mdb_env_close(lmdb->env);
+ lmdb->env = NULL;
+
+ return 0;
+}
+
+static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path)
+{
+ struct lmdb_private *lmdb;
+ int ret;
+
+ lmdb = talloc_zero(ldb, struct lmdb_private);
+ if (lmdb == NULL) {
+ return NULL;
+ }
+ lmdb->ldb = ldb;
+
+ ret = mdb_env_create(&lmdb->env);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not create MDB environment %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+ ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ if (ret != 0) {
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ mdb_env_set_maxreaders(lmdb->env, 100000);
+ /* MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+ ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not open DB %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return NULL;
+ }
+
+ return lmdb;
+
+}
+
+static int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module)
+{
+ const char *path = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct ltdb_private *ltdb = NULL;
+
+ /*
+ * We hold locks, so we must use a private event context
+ * on each returned handle
+ */
+ ldb_set_require_private_event_context(ldb);
+
+ path = lmdb_get_path(url);
+ if (path == NULL) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb = talloc_zero(ldb, struct ltdb_private);
+ if (!ltdb) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ lmdb = lmdb_pvt_create(ldb, ldb, path);
+ if (lmdb == NULL) {
+ ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb->lmdb_private = lmdb;
+ if (flags & LDB_FLG_RDONLY) {
+ ltdb->read_only = true;
+ }
+ return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
+
+
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 00000000000..18d28e822d7
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,57 @@
+/*
+ ldb database library using mdb back end - transaction operations
+
+ Copyright (C) Jakub Hrozek 2015
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include <lmdb.h>
+
+#include "ldb_private.h"
+
+
+struct lmdb_private {
+ struct ldb_context *ldb;
+ MDB_env *env;
+
+ struct lmdb_trans *txlist;
+
+ struct ldb_mdb_metadata {
+ struct ldb_message *attributes;
+ unsigned seqnum;
+ } *meta;
+ int error;
+ MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+ struct lmdb_trans *next;
+ struct lmdb_trans *prev;
+
+ MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 2235bd47c98..cfabe6676bd 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
struct ltdb_private {
const struct kv_db_ops *kv_ops;
TDB_CONTEXT *tdb;
+ struct lmdb_private *lmdb_private;
unsigned int connect_flags;
unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59eca4..9f6ce27d887 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
#include <ldb.h>
#include <ldb_private.h>
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
static struct ldb_context *ldb;
bool show_index = false;
bool validate_contents = false;
@@ -166,6 +171,113 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
}
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ int ret;
+ struct MDB_env *env = NULL;
+ struct MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct MDB_cursor *cursor = NULL;
+ struct MDB_val key;
+ struct MDB_val data;
+
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not create MDB environment: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_env_open(env, fname, MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY, 0644);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open environment for %s: (%d) %s\n",
+ fname,
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not start transaction: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open database: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_open(txn, dbi, &cursor);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open cursor: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not find first record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ while (ret != MDB_NOTFOUND) {
+ struct TDB_DATA tkey = {
+ .dptr = key.mv_data,
+ .dsize = key.mv_size
+ };
+ struct TDB_DATA tdata = {
+ .dptr = data.mv_data,
+ .dsize = data.mv_size
+ };
+ traverse_fn(NULL, tkey, tdata, dn);
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not read next record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ }
+ ret = 0;
+
+close_cursor:
+ mdb_cursor_close(cursor);
+close_txn:
+ mdb_txn_commit(txn);
+close_env:
+ mdb_env_close(env);
+
+ if (ret != 0) {
+ return 1;
+ }
+ return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ /* not built with lmdb support */
+ return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
static void usage( void)
{
printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +341,14 @@ static void usage( void)
fname = argv[optind];
- return dump_tdb(fname, dn, emergency);
+ rc = dump_lmdb(fname, dn, emergency);
+ if (rc != 0) {
+ rc = dump_tdb(fname, dn, emergency);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to open %s\n", fname);
+ return 1;
+ }
+ }
+ return 0;
+
}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 1455f92eb2e..c5cb366de47 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
srcdir = srcdir + '/..'
sys.path.insert(0, srcdir + '/buildtools/wafsamba')
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -30,6 +30,10 @@ def set_options(opt):
opt.RECURSE('lib/tevent')
opt.RECURSE('lib/replace')
opt.tool_options('python') # options for disabling pyc or pyo compilation
+ opt.add_option('--with-lmdb',
+ help=("Build with support for lmdb databases," +
+ "this is an experimental feature"),
+ action="store_true", dest='with_lmdb')
def configure(conf):
conf.RECURSE('lib/tdb')
@@ -110,6 +114,30 @@ def configure(conf):
if not sys.platform.startswith("openbsd"):
conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
+ # if lmdb support is enabled (off by default) then we require lmdb
+ # is present, build the mdb back end and enable lmdb support in
+ # the tools.
+ if Options.options.with_lmdb:
+ if not conf.CHECK_CFG(package='lmdb',
+ args='"lmdb >= 0.9.16" --cflags --libs',
+ msg='Checking for lmdb >= 0.9.16',
+ mandatory=False):
+ if not conf.CHECK_CODE('''
+ #if MDB_VERSION_MAJOR == 0 \
+ && MDB_VERSION_MINOR <= 9 \
+ && MDB_VERSION_PATCH < 16
+ #error LMDB too old
+ #endif
+ ''',
+ 'HAVE_GOOD_LMDB_VERSION',
+ headers='lmdb.h',
+ msg='Checking for lmdb >= 0.9.16 via header check'):
+ raise Utils.WafError('--enable-lmdb requires lmdb 0.9.16 or later')
+
+ if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+ conf.DEFINE('HAVE_LMDB', '1')
+ conf.env.ENABLE_MDB_BACKEND = True
+
conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
conf.SAMBA_CONFIG_H()
@@ -321,6 +349,20 @@ def build(bld):
private_library=True,
deps='tdb ldb')
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_MODULE('ldb_mdb',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ init_function='ldb_mdb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value lmdb',
+ subsystem='ldb')
+ lmdb_deps = ' ldb_mdb_int'
+ else:
+ lmdb_deps = ''
+
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
@@ -338,8 +380,14 @@ def build(bld):
bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ lmdb_deps = ' lmdb'
+ else:
+ lmdb_deps = ''
# ldbdump doesn't get installed
- bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+ bld.SAMBA_BINARY('ldbdump',
+ 'tools/ldbdump.c',
+ deps='ldb-cmdline ldb' + lmdb_deps,
install=False)
bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -353,6 +401,13 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+ source='tests/ldb_mod_op_test.c',
+ cflags='-DTEST_BE=\"mdb\"',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
deps='cmocka ldb',
@@ -385,8 +440,10 @@ def test(ctx):
print("Python testsuite returned %d" % pyret)
cmocka_ret = 0
- for test_exe in ['ldb_tdb_mod_op_test',
- 'ldb_msg_test']:
+ test_exes = ['ldb_tdb_mod_op_test', 'ldb_msg_test']
+ if env.ENABLE_MDB_BACKEND:
+ test_exes.append('ldb_mdb_mod_op_test')
+ for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
diff --git a/wscript b/wscript
index 0985aa94867..792f4046a0c 100644
--- a/wscript
+++ b/wscript
@@ -88,6 +88,10 @@ def set_options(opt):
opt.add_option('--without-relro',
help=("Disable RELRO builds"),
action="store_false", dest='enable_relro')
+ opt.add_option('--with-lmdb',
+ help=("Enable lmdb databases," +
+ "this is an experimental feature"),
+ action="store_true", dest='enable_lmdb')
gr = opt.option_group('developer options')
@@ -184,6 +188,12 @@ def configure(conf):
else:
conf.define('USING_SYSTEM_PAM_WRAPPER', 1)
+ if Options.options.enable_lmdb:
+ conf.env['ENABLE_LMDB'] = True
+ else:
+ # TODO this should default to false in the final release
+ # REJECT if in final patch set
+ conf.env['ENABLE_LMDB'] = False
conf.RECURSE('lib/ldb')
if conf.CHECK_LDFLAGS(['-Wl,--wrap=test']):
--
2.14.3
From 54392bde4f30f91f83df8d8d51faa0a39a7ae734 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 03/20] ldb: Handle multiple ldb url schemes
Here we use ldb_relative_path more often and allow for the stripping of
ldb:// and mdb://.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb-samba/ldb_wrap.c | 6 ++++++
source4/dsdb/samdb/ldb_modules/partition_metadata.c | 20 +++++++++-----------
source4/dsdb/samdb/ldb_modules/schema_load.c | 19 +++++--------------
3 files changed, 20 insertions(+), 25 deletions(-)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index 9959b04ed95..1c01a3921c9 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -361,6 +361,12 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
if (strncmp("tdb://", base_url, 6) == 0) {
base_url = base_url+6;
}
+ else if (strncmp("mdb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
+ else if (strncmp("ldb://", base_url, 6) == 0) {
+ base_url = base_url+6;
+ }
path = talloc_strdup(mem_ctx, base_url);
if (path == NULL) {
return NULL;
diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
index e3ad0d8c6c2..510228e319b 100644
--- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c
+++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c
@@ -18,6 +18,7 @@
*/
#include "dsdb/samdb/ldb_modules/partition.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
#define LDB_METADATA_SEQ_NUM "SEQ_NUM"
@@ -185,7 +186,6 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
TALLOC_CTX *tmp_ctx;
struct partition_private_data *data;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename, *dirname;
int open_flags;
struct stat statbuf;
@@ -202,15 +202,11 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
+
if (!filename) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
@@ -222,7 +218,9 @@ static int partition_metadata_open(struct ldb_module *module, bool create)
/* While provisioning, sam.ldb.d directory may not exist,
* so create it. Ignore errors, if it already exists. */
- dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name);
+ dirname = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d");
if (!dirname) {
talloc_free(tmp_ctx);
return ldb_oom(ldb);
diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c
index 2099fac1159..e21a5a5a7e2 100644
--- a/source4/dsdb/samdb/ldb_modules/schema_load.c
+++ b/source4/dsdb/samdb/ldb_modules/schema_load.c
@@ -32,6 +32,7 @@
#include <tdb.h>
#include "lib/tdb_wrap/tdb_wrap.h"
#include "dsdb/samdb/ldb_modules/util.h"
+#include "lib/ldb-samba/ldb_wrap.h"
#include "system/filesys.h"
struct schema_load_private_data {
@@ -58,7 +59,6 @@ static int schema_metadata_open(struct ldb_module *module)
struct ldb_context *ldb = ldb_module_get_ctx(module);
TALLOC_CTX *tmp_ctx;
struct loadparm_context *lp_ctx;
- const char *sam_name;
char *filename;
int open_flags;
struct stat statbuf;
@@ -74,19 +74,10 @@ static int schema_metadata_open(struct ldb_module *module)
return ldb_module_oom(module);
}
- sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url");
- if (!sam_name) {
- talloc_free(tmp_ctx);
- return ldb_operr(ldb);
- }
- if (strncmp("tdb://", sam_name, 6) == 0) {
- sam_name += 6;
- }
- filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name);
- if (!filename) {
- talloc_free(tmp_ctx);
- return ldb_oom(ldb);
- }
+ /* the backend LDB is the DN (base64 encoded if not 'plain') followed by .ldb */
+ filename = ldb_relative_path(ldb,
+ tmp_ctx,
+ "sam.ldb.d/metadata.tdb");
open_flags = O_RDWR;
if (stat(filename, &statbuf) != 0) {
--
2.14.3
From 93ff96663539a7bb4e3c21763e3138af7452a428 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 04/20] ldb: Introduce new ldb:// prefix to allow failover
If you try to open with tdb first you find that you can clobber a
database due to the default tdb behaviour.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_ldb/ldb_ldb.c | 60 ++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.c | 79 +++++++++++++++++++++++++-----------------
lib/ldb/ldb_mdb/ldb_mdb.h | 3 ++
lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 +++++++++++++++++
lib/ldb/wscript | 19 ++++++++--
5 files changed, 158 insertions(+), 34 deletions(-)
create mode 100644 lib/ldb/ldb_ldb/ldb_ldb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c
diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
new file mode 100644
index 00000000000..14ab51cda10
--- /dev/null
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -0,0 +1,60 @@
+
+#include "ldb_private.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+
+/*
+ connect to the database
+*/
+static int lldb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **module)
+{
+ const char *path;
+ int ret;
+
+ /*
+ * Check and remove the url prefix
+ */
+ if (strchr(url, ':')) {
+ if (strncmp(url, "ldb://", 6) != 0) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR,
+ "Invalid ldb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ path = url+6;
+ } else {
+ path = url;
+ }
+
+ /*
+ * Don't create the database if it's not there
+ */
+ flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+ /*
+ * Try opening the database as an lmdb
+ */
+ ret = lmdb_connect(ldb, path, flags, options, module);
+ if (ret == LDB_SUCCESS) {
+ return ret;
+ }
+ if (ret == LDB_ERR_UNAVAILABLE) {
+ /*
+ * Not mbd to try as tdb
+ */
+ ret = ltdb_connect(ldb, path, flags, options, module);
+ }
+#else
+ ret = ltdb_connect(ldb, path, flags, options, module);
+#endif
+ return ret;
+}
+
+int ldb_ldb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("ldb", lldb_connect, false);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f4648e9e2ec..002913636f2 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -613,26 +613,30 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
return 0;
}
-static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
- struct ldb_context *ldb,
- const char *path)
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags,
+ struct lmdb_private *lmdb)
{
- struct lmdb_private *lmdb;
int ret;
+ unsigned int mdb_flags;
- lmdb = talloc_zero(ldb, struct lmdb_private);
- if (lmdb == NULL) {
- return NULL;
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
}
- lmdb->ldb = ldb;
ret = mdb_env_create(&lmdb->env);
if (ret != 0) {
- ldb_asprintf_errstring(ldb,
- "Could not create MDB environment %s: %s\n",
- path, mdb_strerror(ret));
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
}
/* Close when lmdb is released */
@@ -640,34 +644,45 @@ static struct lmdb_private *lmdb_pvt_create(TALLOC_CTX *mem_ctx,
ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
if (ret != 0) {
- talloc_free(lmdb);
- return NULL;
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not open MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return ldb_mdb_err_map(ret);
}
mdb_env_set_maxreaders(lmdb->env, 100000);
/* MDB_NOSUBDIR implies there is a separate file called path and a
* separate lockfile called path-lock
*/
- ret = mdb_env_open(lmdb->env, path, MDB_NOSUBDIR|MDB_NOTLS, 0644);
+ mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ if (flags & LDB_FLG_RDONLY) {
+ mdb_flags |= MDB_RDONLY;
+ }
+ ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
talloc_free(lmdb);
- return NULL;
+ return ldb_mdb_err_map(ret);
}
- return lmdb;
+ return LDB_SUCCESS;
}
-static int lmdb_connect(struct ldb_context *ldb, const char *url,
- unsigned int flags, const char *options[],
- struct ldb_module **_module)
+int lmdb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **_module)
{
const char *path = NULL;
struct lmdb_private *lmdb = NULL;
struct ltdb_private *ltdb = NULL;
+ int ret;
/*
* We hold locks, so we must use a private event context
@@ -686,12 +701,19 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
ldb_oom(ldb);
return LDB_ERR_OPERATIONS_ERROR;
}
- ltdb->kv_ops = &lmdb_key_value_ops;
- lmdb = lmdb_pvt_create(ldb, ldb, path);
+ lmdb = talloc_zero(ldb, struct lmdb_private);
if (lmdb == NULL) {
- ldb_asprintf_errstring(ldb, "Failed to connect to %s with lmdb", path);
- return LDB_ERR_OPERATIONS_ERROR;
+ TALLOC_FREE(ltdb);
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ lmdb->ldb = ldb;
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ if (ret != LDB_SUCCESS) {
+ return ret;
}
ltdb->lmdb_private = lmdb;
@@ -701,10 +723,3 @@ static int lmdb_connect(struct ldb_context *ldb, const char *url,
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
-
-
-int ldb_mdb_init(const char *version)
-{
- LDB_MODULE_CHECK_VERSION(version);
- return ldb_register_backend("mdb", lmdb_connect, false);
-}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 18d28e822d7..e62e353b54e 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -53,5 +53,8 @@ struct lmdb_trans {
};
int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module);
#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 00000000000..339c3f22b2a
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index c5cb366de47..51575b6856c 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -352,17 +352,32 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_MODULE('ldb_mdb',
bld.SUBDIR('ldb_mdb',
- '''ldb_mdb.c '''),
+ '''ldb_mdb_init.c'''),
init_function='ldb_mdb_init',
module_init_name='ldb_init_module',
internal_module=False,
- deps='ldb ldb_key_value lmdb',
+ deps='ldb ldb_key_value ldb_mdb_int',
subsystem='ldb')
+
+ bld.SAMBA_LIBRARY('ldb_mdb_int',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ private_library=True,
+ deps='ldb lmdb ldb_key_value')
lmdb_deps = ' ldb_mdb_int'
else:
lmdb_deps = ''
+ bld.SAMBA_MODULE('ldb_ldb',
+ bld.SUBDIR('ldb_ldb',
+ '''ldb_ldb.c'''),
+ init_function='ldb_ldb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value' + lmdb_deps,
+ subsystem='ldb')
+
# have a separate subsystem for common/ldb.c, so it can rebuild
# for install with a different -DLDB_MODULESDIR=
bld.SAMBA_SUBSYSTEM('LIBLDB_MAIN',
--
2.14.3
From fa06a7ef1ad48e23fcdfcf69b17b4daffa3e40b1 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 5 Jan 2018 09:28:54 +1300
Subject: [PATCH 05/20] tests: Replace some references to tdb with ldb://
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
python/samba/tests/encrypted_secrets.py | 2 +-
source4/torture/drs/python/samba_tool_drs.py | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/python/samba/tests/encrypted_secrets.py b/python/samba/tests/encrypted_secrets.py
index 3b6934f0ccb..114f3b91108 100644
--- a/python/samba/tests/encrypted_secrets.py
+++ b/python/samba/tests/encrypted_secrets.py
@@ -54,7 +54,7 @@ class EncryptedSecretsTests(TestCase):
backend_subpath = os.path.join("sam.ldb.d",
backend_filename)
backend_path = self.lp.private_path(backend_subpath)
- backenddb = ldb.Ldb(backend_path)
+ backenddb = ldb.Ldb("ldb://" + backend_path, flags=ldb.FLG_DONT_CREATE_DB)
dn = "CN=Administrator,CN=Users,%s" % basedn
diff --git a/source4/torture/drs/python/samba_tool_drs.py b/source4/torture/drs/python/samba_tool_drs.py
index fcc681dc9e4..502c8096603 100644
--- a/source4/torture/drs/python/samba_tool_drs.py
+++ b/source4/torture/drs/python/samba_tool_drs.py
@@ -282,7 +282,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
@@ -291,7 +291,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertEqual(ds_name, server_ds_name)
self.assertEqual(ldap_service_name, server_ldap_service_name)
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
def get_krbtgt_pw():
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
@@ -346,13 +346,13 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.dc1,
self.cmdline_creds,
self.tempdir))
- ldb_rootdse = self._get_rootDSE("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
+ ldb_rootdse = self._get_rootDSE("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"), ldap_only=False)
nc_name = ldb_rootdse["defaultNamingContext"]
config_nc_name = ldb_rootdse["configurationNamingContext"]
ds_name = ldb_rootdse["dsServiceName"]
ldap_service_name = str(server_rootdse["ldapServiceName"][0])
- samdb = samba.tests.connect_samdb("tdb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
+ samdb = samba.tests.connect_samdb("ldb://" + os.path.join(self.tempdir, "private", "sam.ldb"),
ldap_only=False, lp=self.get_loadparm())
krbtgt_pw = samdb.searchone("unicodePwd", "cn=krbtgt,CN=users,%s" % nc_name)
self.assertIsNotNone(krbtgt_pw)
@@ -381,7 +381,7 @@ class SambaToolDrsTests(drs_base.DrsBaseTestCase):
self.assertRaises(samba.tests.BlackboxProcessError, demote_self)
# While we have this cloned, try demoting the other server on the clone
- out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H %s/private/sam.ldb"
+ out = self.check_output("samba-tool domain demote --remove-other-dead-server=%s -H ldb://%s/private/sam.ldb"
% (self.dc2,
self.tempdir))
--
2.14.3
From ee6470588d4642e6030a6be8ede8bfb629939d54 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 11 Jan 2018 12:40:01 +1300
Subject: [PATCH 06/20] tests/dlz_bind9: support for multiple db types
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
source4/torture/dns/dlz_bind9.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source4/torture/dns/dlz_bind9.c b/source4/torture/dns/dlz_bind9.c
index 2234e7a23f6..42b104e070c 100644
--- a/source4/torture/dns/dlz_bind9.c
+++ b/source4/torture/dns/dlz_bind9.c
@@ -58,7 +58,7 @@ static char *test_dlz_bind9_binddns_dir(struct torture_context *tctx,
const char *file)
{
return talloc_asprintf(tctx,
- "%s/%s",
+ "ldb://%s/%s",
lpcfg_binddns_dir(tctx->lp_ctx),
file);
}
--
2.14.3
From 3a843fee0225d5b543f448c9e8129261a2649e9b Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 8 Jan 2018 08:56:16 +1300
Subject: [PATCH 07/20] provision: Allow provisioning of a different database
backend
This sets the backendStore field in @PARTITION, depending on which
argument you set in the provision.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
python/samba/netcmd/domain.py | 13 ++++++++---
python/samba/provision/__init__.py | 41 ++++++++++++++++++++++++---------
python/samba/provision/backend.py | 2 +-
python/samba/provision/sambadns.py | 26 +++++++++++++++------
python/samba/samdb.py | 3 +++
source4/scripting/bin/samba_upgradedns | 3 +--
source4/setup/provision_partitions.ldif | 1 +
7 files changed, 65 insertions(+), 24 deletions(-)
diff --git a/python/samba/netcmd/domain.py b/python/samba/netcmd/domain.py
index d3b9b0ccf6f..3fe08e8588f 100644
--- a/python/samba/netcmd/domain.py
+++ b/python/samba/netcmd/domain.py
@@ -42,7 +42,7 @@ from samba.net import Net, LIBNET_JOIN_AUTOMATIC
import samba.ntacls
from samba.join import join_RODC, join_DC, join_subdomain
from samba.auth import system_session
-from samba.samdb import SamDB
+from samba.samdb import SamDB, get_default_backend_store
from samba.ndr import ndr_unpack, ndr_pack, ndr_print
from samba.dcerpc import drsuapi
from samba.dcerpc import drsblobs
@@ -257,6 +257,9 @@ class cmd_domain_provision(Command):
Option("--plaintext-secrets", action="store_true",
help="Store secret/sensitive values as plain text on disk" +
"(default is to encrypt secret/ensitive values)"),
+ Option("--backend-store", type="choice", metavar="BACKENDSTORE",
+ choices=["tdb", "mdb"],
+ help="Specify the database backend to be used TODO better words"),
]
openldap_options = [
@@ -327,7 +330,8 @@ class cmd_domain_provision(Command):
ldap_backend_forced_uri=None,
ldap_dryrun_mode=None,
base_schema=None,
- plaintext_secrets=False):
+ plaintext_secrets=False,
+ backend_store=None):
self.logger = self.get_logger("provision")
if quiet:
@@ -475,6 +479,8 @@ class cmd_domain_provision(Command):
domain_sid = security.dom_sid(domain_sid)
session = system_session()
+ if backend_store is None:
+ backend_store = get_default_backend_store()
try:
result = provision(self.logger,
session, smbconf=smbconf, targetdir=targetdir,
@@ -497,7 +503,8 @@ class cmd_domain_provision(Command):
ldap_backend_forced_uri=ldap_backend_forced_uri,
nosync=ldap_backend_nosync, ldap_dryrun_mode=ldap_dryrun_mode,
base_schema=base_schema,
- plaintext_secrets=plaintext_secrets)
+ plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
except ProvisioningError as e:
raise CommandError("Provision failed", e)
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index 24c2d55553d..a8f508480e4 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -123,6 +123,7 @@ from samba.schema import Schema
from samba.samdb import SamDB
from samba.dbchecker import dbcheck
from samba.provision.kerberos import create_kdc_conf
+from samba.samdb import get_default_backend_store
DEFAULT_POLICY_GUID = "31B2F340-016D-11D2-945F-00C04FB984F9"
DEFAULT_DC_POLICY_GUID = "6AC1786C-016F-11D2-945F-00C04FB984F9"
@@ -814,7 +815,8 @@ def setup_name_mappings(idmap, sid, root_uid, nobody_uid,
def setup_samdb_partitions(samdb_path, logger, lp, session_info,
provision_backend, names, serverrole,
- erase=False, plaintext_secrets=False):
+ erase=False, plaintext_secrets=False,
+ backend_store=None):
"""Setup the partitions for the SAM database.
Alternatively, provision() may call this, and then populate the database.
@@ -847,11 +849,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
if not plaintext_secrets:
required_features = "requiredFeatures: encryptedSecrets"
+ if backend_store is None:
+ backend_store = get_default_backend_store()
+ backend_store_line = "backendStore: %s" % backend_store
+
samdb.transaction_start()
try:
logger.info("Setting up sam.ldb partitions and settings")
setup_add_ldif(samdb, setup_path("provision_partitions.ldif"), {
- "LDAP_BACKEND_LINE": ldap_backend_line
+ "LDAP_BACKEND_LINE": ldap_backend_line,
+ "BACKEND_STORE": backend_store_line
})
@@ -1245,7 +1252,7 @@ def create_default_gpo(sysvolpath, dnsdomain, policyguid, policyguid_dc):
def setup_samdb(path, session_info, provision_backend, lp, names,
logger, fill, serverrole, schema, am_rodc=False,
- plaintext_secrets=False):
+ plaintext_secrets=False, backend_store=None):
"""Setup a complete SAM Database.
:note: This will wipe the main SAM database file!
@@ -1254,7 +1261,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
# Also wipes the database
setup_samdb_partitions(path, logger=logger, lp=lp,
provision_backend=provision_backend, session_info=session_info,
- names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets)
+ names=names, serverrole=serverrole, plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
# Load the database, but don's load the global schema and don't connect
# quite yet
@@ -1293,7 +1301,8 @@ def setup_samdb(path, session_info, provision_backend, lp, names,
def fill_samdb(samdb, lp, names, logger, policyguid,
policyguid_dc, fill, adminpass, krbtgtpass, machinepass, dns_backend,
dnspass, invocationid, ntdsguid, serverrole, am_rodc=False,
- dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None):
+ dom_for_fun_level=None, schema=None, next_rid=None, dc_rid=None,
+ backend_store=None):
if next_rid is None:
next_rid = 1000
@@ -1831,7 +1840,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
invocationid=None, machinepass=None, ntdsguid=None,
dns_backend=None, dnspass=None,
serverrole=None, dom_for_fun_level=None,
- am_rodc=False, lp=None, use_ntvfs=False, skip_sysvolacl=False):
+ am_rodc=False, lp=None, use_ntvfs=False,
+ skip_sysvolacl=False, backend_store=None):
# create/adapt the group policy GUIDs
# Default GUID for default policy are described at
# "How Core Group Policy Works"
@@ -1863,7 +1873,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
dns_backend=dns_backend, dnspass=dnspass,
ntdsguid=ntdsguid, serverrole=serverrole,
dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
- next_rid=next_rid, dc_rid=dc_rid)
+ next_rid=next_rid, dc_rid=dc_rid,
+ backend_store=backend_store)
# Set up group policies (domain policy and domain controller
# policy)
@@ -1913,7 +1924,8 @@ def provision_fill(samdb, secrets_ldb, logger, names, paths,
setup_ad_dns(samdb, secrets_ldb, names, paths, lp, logger,
hostip=hostip, hostip6=hostip6, dns_backend=dns_backend,
dnspass=dnspass, os_level=dom_for_fun_level,
- targetdir=targetdir, fill_level=samdb_fill)
+ targetdir=targetdir, fill_level=samdb_fill,
+ backend_store=backend_store)
domainguid = samdb.searchone(basedn=samdb.get_default_basedn(),
attribute="objectGUID")
@@ -2018,6 +2030,9 @@ def directory_create_or_exists(path, mode=0o755):
else:
raise ProvisioningError("Failed to create directory %s: %s" % (path, e.strerror))
+# TODO
+# Decide and set the correct backend type, currently mdb for testing
+# but should probably be tdb for release.
def provision(logger, session_info, smbconf=None,
targetdir=None, samdb_fill=FILL_FULL, realm=None, rootdn=None,
domaindn=None, schemadn=None, configdn=None, serverdn=None,
@@ -2033,7 +2048,7 @@ def provision(logger, session_info, smbconf=None,
use_rfc2307=False, maxuid=None, maxgid=None, skip_sysvolacl=True,
ldap_backend_forced_uri=None, nosync=False, ldap_dryrun_mode=False,
ldap_backend_extra_port=None, base_schema=None,
- plaintext_secrets=False):
+ plaintext_secrets=False, backend_store=None):
"""Provision samba4
:note: caution, this wipes all existing data!
@@ -2050,6 +2065,8 @@ def provision(logger, session_info, smbconf=None,
if backend_type is None:
backend_type = "ldb"
+ if backend_store is None:
+ backend_store = get_default_backend_store()
if domainsid is None:
domainsid = security.random_sid()
@@ -2233,7 +2250,8 @@ def provision(logger, session_info, smbconf=None,
provision_backend, lp, names, logger=logger,
serverrole=serverrole,
schema=schema, fill=samdb_fill, am_rodc=am_rodc,
- plaintext_secrets=plaintext_secrets)
+ plaintext_secrets=plaintext_secrets,
+ backend_store=backend_store)
if serverrole == "active directory domain controller":
if paths.netlogon is None:
@@ -2264,7 +2282,8 @@ def provision(logger, session_info, smbconf=None,
dnspass=dnspass, serverrole=serverrole,
dom_for_fun_level=dom_for_fun_level, am_rodc=am_rodc,
lp=lp, use_ntvfs=use_ntvfs,
- skip_sysvolacl=skip_sysvolacl)
+ skip_sysvolacl=skip_sysvolacl,
+ backend_store=backend_store)
if not is_heimdal_built():
create_kdc_conf(paths.kdcconf, realm, domain, os.path.dirname(lp.get("log file")))
diff --git a/python/samba/provision/backend.py b/python/samba/provision/backend.py
index 278e69f649d..e457dd0e301 100644
--- a/python/samba/provision/backend.py
+++ b/python/samba/provision/backend.py
@@ -145,7 +145,7 @@ class ExistingBackend(ProvisionBackend):
def init(self):
# Check to see that this 'existing' LDAP backend in fact exists
- ldapi_db = Ldb(self.ldapi_uri)
+ ldapi_db = Ldb(self.ldapi_uri, flags=Ldb.FLG_DONT_CREATE_DB)
ldapi_db.search(base="", scope=SCOPE_BASE,
expression="(objectClass=OpenLDAProotDSE)")
diff --git a/python/samba/provision/sambadns.py b/python/samba/provision/sambadns.py
index 4cc15b06700..13f2197a8d7 100644
--- a/python/samba/provision/sambadns.py
+++ b/python/samba/provision/sambadns.py
@@ -58,6 +58,7 @@ from samba.provision.common import (
FILL_DRS,
)
+from samba.samdb import get_default_backend_store
def get_domainguid(samdb, domaindn):
res = samdb.search(base=domaindn, scope=ldb.SCOPE_BASE, attrs=["objectGUID"])
@@ -787,12 +788,19 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
# Find the partitions and corresponding filenames
partfile = {}
- res = samdb.search(base="@PARTITION", scope=ldb.SCOPE_BASE, attrs=["partition"])
+ res = samdb.search(base="@PARTITION",
+ scope=ldb.SCOPE_BASE,
+ attrs=["partition", "backendStore"])
for tmp in res[0]["partition"]:
(nc, fname) = tmp.split(':')
partfile[nc.upper()] = fname
+ backend_store = get_default_backend_store()
+ if "backendStore" in res[0]:
+ backend_store = res[0]["backendStore"]
+
# Create empty domain partition
+
domaindn = names.domaindn.upper()
domainpart_file = os.path.join(dns_dir, partfile[domaindn])
try:
@@ -800,7 +808,8 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
file(domainpart_file, 'w').close()
# Fill the basedn and @OPTION records in domain partition
- dom_ldb = samba.Ldb(domainpart_file)
+ dom_url = "%s://%s" % (backend_store, domainpart_file)
+ dom_ldb = samba.Ldb(dom_url)
domainguid_line = "objectGUID: %s\n-" % domainguid
descr = b64encode(get_domain_descriptor(domainsid))
setup_add_ldif(dom_ldb, setup_path("provision_basedn.ldif"), {
@@ -875,7 +884,7 @@ def create_samdb_copy(samdb, logger, paths, names, domainsid, domainguid):
os.chown(dpath, -1, paths.bind_gid)
os.chmod(dpath, 0o770)
for f in files:
- if f.endswith('.ldb') or f.endswith('.tdb'):
+ if f.endswith(('.ldb', '.tdb', 'ldb-lock')):
fpath = os.path.join(dirname, f)
os.chown(fpath, -1, paths.bind_gid)
os.chmod(fpath, 0o660)
@@ -1069,7 +1078,7 @@ def fill_dns_data_partitions(samdb, domainsid, site, domaindn, forestdn,
def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, dnspass=None, hostip=None, hostip6=None,
- targetdir=None, fill_level=FILL_FULL):
+ targetdir=None, fill_level=FILL_FULL, backend_store="mdb"):
"""Provision DNS information (assuming GC role)
:param samdb: LDB object connected to sam.ldb file
@@ -1164,12 +1173,14 @@ def setup_ad_dns(samdb, secretsdb, names, paths, lp, logger,
if dns_backend.startswith("BIND9_"):
setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, site=site, dnspass=dnspass, hostip=hostip,
- hostip6=hostip6, targetdir=targetdir)
+ hostip6=hostip6, targetdir=targetdir,
+ backend_store=backend_store)
def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
dns_backend, os_level, site=None, dnspass=None, hostip=None,
- hostip6=None, targetdir=None, key_version_number=None):
+ hostip6=None, targetdir=None, key_version_number=None,
+ backend_store=None):
"""Provision DNS information (assuming BIND9 backend in DC role)
:param samdb: LDB object connected to sam.ldb file
@@ -1216,7 +1227,8 @@ def setup_bind9_dns(samdb, secretsdb, names, paths, lp, logger,
ntdsguid=names.ntdsguid)
if dns_backend == "BIND9_DLZ" and os_level >= DS_DOMAIN_FUNCTION_2003:
- create_samdb_copy(samdb, logger, paths, names, names.domainsid, domainguid)
+ create_samdb_copy(samdb, logger, paths,
+ names, names.domainsid, domainguid)
create_named_conf(paths, realm=names.realm,
dnsdomain=names.dnsdomain, dns_backend=dns_backend,
diff --git a/python/samba/samdb.py b/python/samba/samdb.py
index 89014a53ed2..c51864bebd7 100644
--- a/python/samba/samdb.py
+++ b/python/samba/samdb.py
@@ -36,6 +36,9 @@ from samba.compat import text_type
__docformat__ = "restructuredText"
+def get_default_backend_store():
+ return "tdb"
+
class SamDB(samba.Ldb):
"""The SAM database."""
diff --git a/source4/scripting/bin/samba_upgradedns b/source4/scripting/bin/samba_upgradedns
index db15b65e1dc..3a43577f244 100755
--- a/source4/scripting/bin/samba_upgradedns
+++ b/source4/scripting/bin/samba_upgradedns
@@ -533,8 +533,7 @@ if __name__ == '__main__':
create_dns_dir(logger, paths)
# Setup a copy of SAM for BIND9
- create_samdb_copy(ldbs.sam, logger, paths, names, domainsid,
- domainguid)
+ create_samdb_copy(ldbs.sam, logger, paths, names, domainsid, domainguid)
create_named_conf(paths, names.realm, dnsdomain, opts.dns_backend, logger)
diff --git a/source4/setup/provision_partitions.ldif b/source4/setup/provision_partitions.ldif
index 728f32739d4..4cd57943228 100644
--- a/source4/setup/provision_partitions.ldif
+++ b/source4/setup/provision_partitions.ldif
@@ -2,5 +2,6 @@ dn: @PARTITION
replicateEntries: @ATTRIBUTES
replicateEntries: @INDEXLIST
replicateEntries: @OPTIONS
+${BACKEND_STORE}
${LDAP_BACKEND_LINE}
--
2.14.3
From 9d5b6a813fc8a71221c49ee0692e57eca5a3695c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Feb 2018 11:10:34 +1300
Subject: [PATCH 08/20] dsdb: add lmdbLevelOne as a required feature.
---
python/samba/provision/__init__.py | 12 +++++++++++-
source4/dsdb/samdb/ldb_modules/samba_dsdb.c | 5 ++++-
source4/dsdb/samdb/samdb.h | 9 +++++++++
3 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/python/samba/provision/__init__.py b/python/samba/provision/__init__.py
index a8f508480e4..fcb379c54fe 100644
--- a/python/samba/provision/__init__.py
+++ b/python/samba/provision/__init__.py
@@ -845,7 +845,7 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
if provision_backend.type != "ldb":
ldap_backend_line = "ldapBackend: %s" % provision_backend.ldap_uri
- required_features = "# No required features"
+ required_features = None
if not plaintext_secrets:
required_features = "requiredFeatures: encryptedSecrets"
@@ -853,6 +853,16 @@ def setup_samdb_partitions(samdb_path, logger, lp, session_info,
backend_store = get_default_backend_store()
backend_store_line = "backendStore: %s" % backend_store
+ if backend_store == "mdb":
+ if required_features is not None:
+ required_features += "\n"
+ else:
+ required_features = ""
+ required_features += "requiredFeatures: lmdbLevelOne"
+
+ if required_features is None:
+ required_features = "# No required features"
+
samdb.transaction_start()
try:
logger.info("Setting up sam.ldb partitions and settings")
diff --git a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
index e0acb4e371a..2605c1e511e 100644
--- a/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
+++ b/source4/dsdb/samdb/ldb_modules/samba_dsdb.c
@@ -234,8 +234,11 @@ static bool check_required_features(struct ldb_message_element *el)
int k;
DATA_BLOB esf = data_blob_string_const(
SAMBA_ENCRYPTED_SECRETS_FEATURE);
+ DATA_BLOB lmdbl1 = data_blob_string_const(
+ SAMBA_LMDB_LEVEL_ONE_FEATURE);
for (k = 0; k < el->num_values; k++) {
- if (data_blob_cmp(&esf, &el->values[k]) != 0) {
+ if ((data_blob_cmp(&esf, &el->values[k]) != 0) &&
+ (data_blob_cmp(&lmdbl1, &el->values[k]) != 0)) {
return false;
}
}
diff --git a/source4/dsdb/samdb/samdb.h b/source4/dsdb/samdb/samdb.h
index 6a4820c3378..b5f558ac502 100644
--- a/source4/dsdb/samdb/samdb.h
+++ b/source4/dsdb/samdb/samdb.h
@@ -333,5 +333,14 @@ struct dsdb_extended_sec_desc_propagation_op {
#define SAMBA_SORTED_LINKS_FEATURE "sortedLinks"
#define SAMBA_ENCRYPTED_SECRETS_FEATURE "encryptedSecrets"
+/*
+ * lmdb level one feature is an experimental release with basic support
+ * for lmdb database files, instead of tdb.
+ * - Keys are limited to 511 bytes long so GUID indexes are required
+ * - Currently only the:
+ * partition data files
+ * are in lmdb format.
+ */
+#define SAMBA_LMDB_LEVEL_ONE_FEATURE "lmdbLevelOne"
#endif /* __SAMDB_H__ */
--
2.14.3
From f0b0367609f0204b9a542063e5c6cdfc46c41c52 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 19 Jan 2018 09:28:14 +1300
Subject: [PATCH 09/20] ldb_mod_op_test: Add new nested transactions test
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/ldb_mod_op_test.c | 62 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 5878143d8f6..425a984ed5d 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -626,6 +626,46 @@ static void test_transactions(void **state)
assert_int_equal(res->count, 0);
}
+static void test_nested_transactions(void **state)
+{
+ int ret;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ struct ldb_result *res;
+
+ /* start lev-0 transaction */
+ ret = ldb_transaction_start(test_ctx->ldb);
+ assert_int_equal(ret, 0);
+
+ add_keyval(test_ctx, "vegetable", "carrot");
+
+
+ /* start another lev-1 nested transaction */
+ ret = ldb_transaction_start(test_ctx->ldb);
+ assert_int_equal(ret, 0);
+
+ add_keyval(test_ctx, "fruit", "apple");
+
+ /* abort lev-1 nested transaction */
+ ret = ldb_transaction_cancel(test_ctx->ldb);
+ assert_int_equal(ret, 0);
+
+ /* commit lev-0 transaction */
+ ret = ldb_transaction_commit(test_ctx->ldb);
+ assert_int_equal(ret, 0);
+
+ res = get_keyval(test_ctx, "vegetable", "carrot");
+ assert_non_null(res);
+ assert_int_equal(res->count, 1);
+
+ /* This documents the current ldb behaviour, i.e. nested
+ * transactions are not supported. And the cancellation of the nested
+ * transaction has no effect.
+ */
+ res = get_keyval(test_ctx, "fruit", "apple");
+ assert_non_null(res);
+ assert_int_equal(res->count, 1);
+}
struct ldb_mod_test_ctx {
struct ldbtest_ctx *ldb_test_ctx;
const char *entry_dn;
@@ -3609,6 +3649,21 @@ static void test_ldb_guid_index_duplicate_dn_logging(void **state)
talloc_free(tmp_ctx);
}
+static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
+{
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ ldb_transaction_start(test_ctx->ldb);
+
+ /*
+ * Trigger the destructor
+ */
+ TALLOC_FREE(test_ctx->ldb);
+}
+
int main(int argc, const char **argv)
{
@@ -3643,6 +3698,9 @@ int main(int argc, const char **argv)
cmocka_unit_test_setup_teardown(test_transactions,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(test_nested_transactions,
+ ldbtest_setup,
+ ldbtest_teardown),
cmocka_unit_test_setup_teardown(test_ldb_modify_add_key,
ldb_modify_test_setup,
ldb_modify_test_teardown),
@@ -3765,6 +3823,10 @@ int main(int argc, const char **argv)
test_ldb_unique_index_duplicate_with_guid,
ldb_guid_index_test_setup,
ldb_guid_index_test_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_talloc_destructor_transaction_cleanup,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.14.3
From 1ff7b4548477888178be02203755c5aff52b5a30 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 23 Jan 2018 11:03:16 +1300
Subject: [PATCH 10/20] ldb_mod_op_test: Make sure that closing the database
frees locks
Without the destructor firing, this test used to pass, but now we show
that we must be able to open a new ldb handle.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/ldb_mod_op_test.c | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 425a984ed5d..b91130252a7 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3662,6 +3662,35 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
* Trigger the destructor
*/
TALLOC_FREE(test_ctx->ldb);
+
+ /*
+ * Now ensure that a new connection can be opened
+ */
+ {
+ TALLOC_CTX *tctx = talloc_new(test_ctx);
+ struct ldbtest_ctx *ctx = talloc_zero(tctx, struct ldbtest_ctx);
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+ int ret;
+
+ ldbtest_setup((void *)&ctx);
+
+ basedn = ldb_dn_new_fmt(tctx, ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(ctx->ldb,
+ tctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ assert_int_equal(ret, 0);
+ assert_non_null(result);
+ assert_int_equal(result->count, 0);
+
+ ldbtest_teardown((void *)&ctx);
+ }
}
--
2.14.3
From 3b6bccf7eb0d0f6160b4cf047a03581d430a784e Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Fri, 16 Feb 2018 17:13:26 +1300
Subject: [PATCH 11/20] ldb: Change some prototypes to using ldb_val instead of
TDB_DATA
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 59 +++++++++++++++++++++++----------------------
lib/ldb/ldb_tdb/ldb_index.c | 8 +++---
lib/ldb/ldb_tdb/ldb_tdb.c | 50 ++++++++++++++++++++++++++++++++------
lib/ldb/ldb_tdb/ldb_tdb.h | 8 +++---
4 files changed, 80 insertions(+), 45 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 002913636f2..f05b11bcad3 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -145,8 +145,8 @@ static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
}
static int lmdb_store(struct ltdb_private *ltdb,
- TDB_DATA key,
- TDB_DATA data, int flags)
+ struct ldb_val key,
+ struct ldb_val data, int flags)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
MDB_val mdb_key;
@@ -167,11 +167,11 @@ static int lmdb_store(struct ltdb_private *ltdb,
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
- mdb_data.mv_size = data.dsize;
- mdb_data.mv_data = data.dptr;
+ mdb_data.mv_size = data.length;
+ mdb_data.mv_data = data.data;
if (flags == TDB_INSERT) {
mdb_flags = MDB_NOOVERWRITE;
@@ -202,7 +202,8 @@ static int lmdb_store(struct ltdb_private *ltdb,
return ldb_mdb_err_map(lmdb->error);
}
-static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
MDB_val mdb_key;
@@ -221,8 +222,8 @@ static int lmdb_delete(struct ltdb_private *ltdb, TDB_DATA key)
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
if (lmdb->error != MDB_SUCCESS) {
@@ -298,22 +299,22 @@ done:
}
static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
- TDB_DATA key,
- TDB_DATA key2,
- TDB_DATA data,
+ struct ldb_val key,
+ struct ldb_val key2,
+ struct ldb_val data,
void *state)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
- struct TDB_DATA copy;
+ struct ldb_val copy;
int ret = LDB_SUCCESS;
/*
* Need to take a copy of the data as the delete operation alters the
* data, as it is in private lmdb memory.
*/
- copy.dsize = data.dsize;
- copy.dptr = talloc_memdup(ltdb, data.dptr, data.dsize);
- if (copy.dptr == NULL) {
+ copy.length = data.length;
+ copy.data = talloc_memdup(ltdb, data.data, data.length);
+ if (copy.data == NULL) {
lmdb->error = MDB_PANIC;
return ldb_oom(lmdb->ldb);
}
@@ -325,10 +326,10 @@ static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
LDB_DEBUG_ERROR,
"Failed to delete %*.*s "
"for rekey as %*.*s: %s",
- (int)key.dsize, (int)key.dsize,
- (const char *)key.dptr,
- (int)key2.dsize, (int)key2.dsize,
- (const char *)key.dptr,
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
mdb_strerror(lmdb->error));
ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
goto done;
@@ -339,26 +340,26 @@ static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
lmdb->ldb,
LDB_DEBUG_ERROR,
"Failed to rekey %*.*s as %*.*s: %s",
- (int)key.dsize, (int)key.dsize,
- (const char *)key.dptr,
- (int)key2.dsize, (int)key2.dsize,
- (const char *)key.dptr,
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
mdb_strerror(lmdb->error));
ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
goto done;
}
done:
- if (copy.dptr != NULL) {
- TALLOC_FREE(copy.dptr);
- copy.dsize = 0;
+ if (copy.data != NULL) {
+ TALLOC_FREE(copy.data);
+ copy.length = 0;
}
/*
* Explicity invalidate the data, as the delete has done this
*/
- data.dsize = 0;
- data.dptr = NULL;
+ data.length = 0;
+ data.data = NULL;
return ret;
}
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index bb534c3833c..076db10f2dd 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -2783,11 +2783,11 @@ static int re_key(struct ltdb_private *ltdb, struct ldb_val ldb_key, struct ldb_
}
if (key.dsize != key2.dsize ||
(memcmp(key.dptr, key2.dptr, key.dsize) != 0)) {
- TDB_DATA data = {
- .dptr = val.data,
- .dsize = val.length
+ struct ldb_val ldb_key2 = {
+ .data = key2.dptr,
+ .length = key2.dsize
};
- ltdb->kv_ops->update_in_iterate(ltdb, key, key2, data, ctx);
+ ltdb->kv_ops->update_in_iterate(ltdb, ldb_key, ldb_key2, val, ctx);
}
talloc_free(key2.dptr);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 4f1c241e96a..9ee28bb41fc 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1,4 +1,5 @@
/*
+ *
ldb database library
Copyright (C) Andrew Tridgell 2004
@@ -413,8 +414,17 @@ static int ltdb_modified(struct ldb_module *module, struct ldb_dn *dn)
return ret;
}
-static int ltdb_tdb_store(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags)
+static int ltdb_tdb_store(struct ltdb_private *ltdb, struct ldb_val ldb_key,
+ struct ldb_val ldb_data, int flags)
{
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+ TDB_DATA data = {
+ .dptr = ldb_data.data,
+ .dsize = ldb_data.length
+ };
return tdb_store(ltdb->tdb, key, data, flags);
}
@@ -435,7 +445,8 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
- TDB_DATA tdb_key, tdb_data;
+ TDB_DATA tdb_key;
+ struct ldb_val ldb_key;
struct ldb_val ldb_data;
int ret = LDB_SUCCESS;
TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -461,10 +472,10 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
return LDB_ERR_OTHER;
}
- tdb_data.dptr = ldb_data.data;
- tdb_data.dsize = ldb_data.length;
+ ldb_key.data = tdb_key.dptr;
+ ldb_key.length = tdb_key.dsize;
- ret = ltdb->kv_ops->store(ltdb, tdb_key, tdb_data, flgs);
+ ret = ltdb->kv_ops->store(ltdb, ldb_key, ldb_data, flgs);
if (ret != 0) {
bool is_special = ldb_dn_is_special(msg->dn);
ret = ltdb->kv_ops->error(ltdb);
@@ -650,8 +661,12 @@ static int ltdb_add(struct ltdb_context *ctx)
return ret;
}
-static int ltdb_tdb_delete(struct ltdb_private *ltdb, TDB_DATA tdb_key)
+static int ltdb_tdb_delete(struct ltdb_private *ltdb, struct ldb_val ldb_key)
{
+ TDB_DATA tdb_key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
return tdb_delete(ltdb->tdb, tdb_key);
}
@@ -664,6 +679,7 @@ int ltdb_delete_noindex(struct ldb_module *module,
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct ldb_val ldb_key;
TDB_DATA tdb_key;
int ret;
TALLOC_CTX *tdb_key_ctx = talloc_new(module);
@@ -682,7 +698,10 @@ int ltdb_delete_noindex(struct ldb_module *module,
return LDB_ERR_OTHER;
}
- ret = ltdb->kv_ops->delete(ltdb, tdb_key);
+ ldb_key.data = tdb_key.dptr;
+ ldb_key.length = tdb_key.dsize;
+
+ ret = ltdb->kv_ops->delete(ltdb, ldb_key);
TALLOC_FREE(tdb_key_ctx);
if (ret != 0) {
@@ -1747,12 +1766,27 @@ static int ltdb_tdb_traverse_fn(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn
}
}
-static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA key2, TDB_DATA data, void *state)
+static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
+ struct ldb_val ldb_key,
+ struct ldb_val ldb_key2,
+ struct ldb_val ldb_data, void *state)
{
int tdb_ret;
struct ldb_context *ldb;
struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state;
struct ldb_module *module = ctx->module;
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+ TDB_DATA key2 = {
+ .dptr = ldb_key2.data,
+ .dsize = ldb_key2.length
+ };
+ TDB_DATA data = {
+ .dptr = ldb_data.data,
+ .dsize = ldb_data.length
+ };
ldb = ldb_module_get_ctx(module);
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index cfabe6676bd..92bbd0e93af 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -10,11 +10,11 @@ typedef int (*ldb_kv_traverse_fn)(struct ltdb_private *ltdb,
void *ctx);
struct kv_db_ops {
- int (*store)(struct ltdb_private *ltdb, TDB_DATA key, TDB_DATA data, int flags);
- int (*delete)(struct ltdb_private *ltdb, TDB_DATA key);
+ int (*store)(struct ltdb_private *ltdb, struct ldb_val key, struct ldb_val data, int flags);
+ int (*delete)(struct ltdb_private *ltdb, struct ldb_val key);
int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
- int (*update_in_iterate)(struct ltdb_private *ltdb, TDB_DATA key,
- TDB_DATA key2, TDB_DATA data, void *ctx);
+ int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
+ struct ldb_val key2, struct ldb_val data, void *ctx);
int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
int (*parser)(TDB_DATA key, TDB_DATA data,
void *private_data),
--
2.14.3
From a0fc01f117efe5a04a411a929bdad7636c373926 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 19 Feb 2018 12:37:20 +1300
Subject: [PATCH 12/20] ldb: Change remaining fetch prototypes to remove
TDB_DATA
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 16 ++++++++--------
lib/ldb/ldb_tdb/ldb_search.c | 28 +++++++++++++++-------------
lib/ldb/ldb_tdb/ldb_tdb.c | 38 +++++++++++++++++++++++++++++++++++---
lib/ldb/ldb_tdb/ldb_tdb.h | 4 ++--
4 files changed, 60 insertions(+), 26 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index f05b11bcad3..60909f6b3fc 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -364,9 +364,9 @@ done:
return ret;
}
/* Handles only a single record */
-static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
- void *private_data),
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
+ void *private_data),
void *ctx)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
@@ -374,7 +374,7 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
MDB_val mdb_data;
MDB_txn *txn = NULL;
MDB_dbi dbi;
- TDB_DATA data;
+ struct ldb_val data;
txn = get_current_txn(lmdb);
if (txn == NULL) {
@@ -388,8 +388,8 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- mdb_key.mv_size = key.dsize;
- mdb_key.mv_data = key.dptr;
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
if (lmdb->error != MDB_SUCCESS) {
@@ -404,8 +404,8 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
}
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
- data.dptr = mdb_data.mv_data;
- data.dsize = mdb_data.mv_size;
+ data.data = mdb_data.mv_data;
+ data.length = mdb_data.mv_size;
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c
index 78ef8b0abb1..35583a1eaa3 100644
--- a/lib/ldb/ldb_tdb/ldb_search.c
+++ b/lib/ldb/ldb_tdb/ldb_search.c
@@ -180,17 +180,15 @@ struct ltdb_parse_data_unpack_ctx {
unsigned int unpack_flags;
};
-static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
+static int ltdb_parse_data_unpack(struct ldb_val key,
+ struct ldb_val data,
void *private_data)
{
struct ltdb_parse_data_unpack_ctx *ctx = private_data;
unsigned int nb_elements_in_db;
int ret;
struct ldb_context *ldb = ldb_module_get_ctx(ctx->module);
- struct ldb_val data_parse = {
- .data = data.dptr,
- .length = data.dsize
- };
+ struct ldb_val data_parse = data;
if (ctx->unpack_flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
/*
@@ -200,13 +198,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
* and the caller needs a stable result.
*/
data_parse.data = talloc_memdup(ctx->msg,
- data.dptr,
- data.dsize);
+ data.data,
+ data.length);
if (data_parse.data == NULL) {
ldb_debug(ldb, LDB_DEBUG_ERROR,
"Unable to allocate data(%d) for %*.*s\n",
- (int)data.dsize,
- (int)key.dsize, (int)key.dsize, key.dptr);
+ (int)data.length,
+ (int)key.length, (int)key.length, key.data);
return LDB_ERR_OPERATIONS_ERROR;
}
}
@@ -217,13 +215,13 @@ static int ltdb_parse_data_unpack(TDB_DATA key, TDB_DATA data,
ctx->unpack_flags,
&nb_elements_in_db);
if (ret == -1) {
- if (data_parse.data != data.dptr) {
+ if (data_parse.data != data.data) {
talloc_free(data_parse.data);
}
ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %*.*s\n",
- (int)key.dsize, (int)key.dsize, key.dptr);
- return LDB_ERR_OPERATIONS_ERROR;
+ (int)key.length, (int)key.length, key.data);
+ return LDB_ERR_OPERATIONS_ERROR;
}
return ret;
}
@@ -246,13 +244,17 @@ int ltdb_search_key(struct ldb_module *module, struct ltdb_private *ltdb,
.module = module,
.unpack_flags = unpack_flags
};
+ struct ldb_val ldb_key = {
+ .data = tdb_key.dptr,
+ .length = tdb_key.dsize
+ };
memset(msg, 0, sizeof(*msg));
msg->num_elements = 0;
msg->elements = NULL;
- ret = ltdb->kv_ops->fetch_and_parse(ltdb, tdb_key,
+ ret = ltdb->kv_ops->fetch_and_parse(ltdb, ldb_key,
ltdb_parse_data_unpack, &ctx);
if (ret == -1) {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 9ee28bb41fc..206b272f28b 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1736,6 +1736,9 @@ struct kv_ctx {
ldb_kv_traverse_fn kv_traverse_fn;
void *ctx;
struct ltdb_private *ltdb;
+ int (*parser)(struct ldb_val key,
+ struct ldb_val data,
+ void *private_data);
};
static int ldb_tdb_traverse_fn_wrapper(struct tdb_context *tdb, TDB_DATA tdb_key, TDB_DATA tdb_data, void *ctx)
@@ -1818,12 +1821,41 @@ static int ltdb_tdb_update_in_iterate(struct ltdb_private *ltdb,
return tdb_ret;
}
-static int ltdb_tdb_parse_record(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
+static int ltdb_tdb_parse_record_wrapper(TDB_DATA tdb_key, TDB_DATA tdb_data,
+ void *ctx)
+{
+ struct kv_ctx *kv_ctx = ctx;
+ struct ldb_val key = {
+ .length = tdb_key.dsize,
+ .data = tdb_key.dptr,
+ };
+ struct ldb_val data = {
+ .length = tdb_data.dsize,
+ .data = tdb_data.dptr,
+ };
+
+ return kv_ctx->parser(key, data, kv_ctx->ctx);
+}
+
+static int ltdb_tdb_parse_record(struct ltdb_private *ltdb,
+ struct ldb_val ldb_key,
+ int (*parser)(struct ldb_val key,
+ struct ldb_val data,
void *private_data),
void *ctx)
{
- return tdb_parse_record(ltdb->tdb, key, parser, ctx);
+ struct kv_ctx kv_ctx = {
+ .parser = parser,
+ .ctx = ctx,
+ .ltdb = ltdb
+ };
+ TDB_DATA key = {
+ .dptr = ldb_key.data,
+ .dsize = ldb_key.length
+ };
+
+ return tdb_parse_record(ltdb->tdb, key, ltdb_tdb_parse_record_wrapper,
+ &kv_ctx);
}
static const char * ltdb_tdb_name(struct ltdb_private *ltdb)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 92bbd0e93af..11396eec9f6 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -15,8 +15,8 @@ struct kv_db_ops {
int (*iterate)(struct ltdb_private *ltdb, ldb_kv_traverse_fn fn, void *ctx);
int (*update_in_iterate)(struct ltdb_private *ltdb, struct ldb_val key,
struct ldb_val key2, struct ldb_val data, void *ctx);
- int (*fetch_and_parse)(struct ltdb_private *ltdb, TDB_DATA key,
- int (*parser)(TDB_DATA key, TDB_DATA data,
+ int (*fetch_and_parse)(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
void *private_data),
void *ctx);
int (*lock_read)(struct ldb_module *);
--
2.14.3
From 4063374653171758237fec227bb3752c04fdeb48 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 13/20] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb
This is used in selftest with 'ldb:nosync = true'.
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 60909f6b3fc..07c2339633a 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -661,6 +661,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
if (flags & LDB_FLG_RDONLY) {
mdb_flags |= MDB_RDONLY;
}
+ if (flags & LDB_FLG_NOSYNC) {
+ mdb_flags |= MDB_NOSYNC;
+ }
ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
--
2.14.3
From 7a19c35f20e08d49146d30f839a3e532e23fa32a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 17 Jan 2018 14:02:09 +1300
Subject: [PATCH 14/20] upgradeprovision: Do not copy backup lmdb -lock files
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
source4/scripting/bin/samba_upgradeprovision | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/source4/scripting/bin/samba_upgradeprovision b/source4/scripting/bin/samba_upgradeprovision
index f3e690ba552..e0db12e0818 100755
--- a/source4/scripting/bin/samba_upgradeprovision
+++ b/source4/scripting/bin/samba_upgradeprovision
@@ -1397,9 +1397,10 @@ def backup_provision(paths, dir, only_db):
else:
os.mkdir(os.path.join(dir, "sam.ldb.d"), 0700)
- for ldb in os.listdir(samldbdir):
- tdb_util.tdb_copy(os.path.join(samldbdir, ldb),
- os.path.join(dir, "sam.ldb.d", ldb))
+ for ldb_name in os.listdir(samldbdir):
+ if not ldb_name.endswith("-lock"):
+ tdb_util.tdb_copy(os.path.join(samldbdir, ldb_name),
+ os.path.join(dir, "sam.ldb.d", ldb_name))
def sync_calculated_attributes(samdb, names):
--
2.14.3
From 56a708d643d5f1305a1dde9f783002685393be10 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 15/20] tests: Add tests to check for max key length and DB
size
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/ldb_lmdb_size_test.c | 193 ++++++++++++++++++++++++++++++++
lib/ldb/tests/ldb_lmdb_test.c | 223 +++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 9 ++
3 files changed, 425 insertions(+)
create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
create mode 100644 lib/ldb/tests/ldb_lmdb_test.c
diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 00000000000..179b54ed18c
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,193 @@
+/*
+ * lmdb backend specific tests for ldb
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+ int ret, x;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ const int MB = 1024 * 1024;
+ char *blob = NULL;
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ blob = talloc_zero_size(tmp_ctx, (MB + 1));
+ assert_non_null(blob);
+ memset(blob, 'x', MB);
+
+
+ for (x = 0; x < 6144; x++) {
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+ assert_non_null(msg->dn);
+
+ ldb_transaction_start(test_ctx->ldb);
+ ret = ldb_msg_add_string(msg, "blob", blob);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+ ldb_transaction_commit(test_ctx->ldb);
+
+ TALLOC_FREE(msg);
+ }
+ talloc_free(tmp_ctx);
+ {
+ struct stat s;
+ ret = stat(test_ctx->dbfile, &s);
+ assert_int_equal(ret, 0);
+ assert_true(s.st_size > (6144LL * MB));
+ }
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_db_size_gt_4GB,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 00000000000..00777a84000
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,223 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+
+#define TEVENT_DEPRECATED 1
+#include <tevent.h>
+
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+
+#define TEST_BE "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_eq_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 51575b6856c..98e906250df 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -422,6 +422,14 @@ def build(bld):
cflags='-DTEST_BE=\"mdb\"',
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_lmdb_test',
+ source='tests/ldb_lmdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+ bld.SAMBA_BINARY('ldb_lmdb_size_test',
+ source='tests/ldb_lmdb_size_test.c',
+ deps='cmocka ldb',
+ install=False)
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
@@ -458,6 +466,7 @@ def test(ctx):
test_exes = ['ldb_tdb_mod_op_test', 'ldb_msg_test']
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
+ test_exes.append('ldb_lmdb_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.14.3
From 27ad45997565cdcf699661b23114c5f9835a764b Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 16/20] ldb_mdb: Store pid to change destructor on fork
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 2 ++
2 files changed, 24 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 07c2339633a..24ba56112fa 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -587,6 +587,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
{
struct lmdb_trans *ltx = NULL;
+ /* Check if this is a forked child */
+ if (getpid() != lmdb->pid) {
+ int fd = 0;
+ /*
+ * We cannot call mdb_env_close or commit any transactions,
+ * otherwise they might appear finished in the parent.
+ *
+ */
+
+ if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+ close(fd);
+ }
+
+ /* Remove the pointer, so that no access should occur */
+ lmdb->env = NULL;
+
+ return 0;
+ }
+
/*
* Close the read transaction if it's open
*/
@@ -673,6 +692,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
return ldb_mdb_err_map(ret);
}
+ /* Store the original pid during the LMDB open */
+ lmdb->pid = getpid();
+
return LDB_SUCCESS;
}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index e62e353b54e..7e0729cc7af 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -43,6 +43,8 @@ struct lmdb_private {
int error;
MDB_txn *read_txn;
+ pid_t pid;
+
};
struct lmdb_trans {
--
2.14.3
From 67712e5beb87b44b0d7bea4f4dbf532f354b6012 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 17/20] lib ldb tests: Run api and index test on tdb and lmdb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 228 +++++++++++++++++++++++++++++++-----------
lib/ldb/tests/python/index.py | 40 +++++++-
2 files changed, 210 insertions(+), 58 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 85fe1bc360f..60678c86023 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -12,6 +12,9 @@ import shutil
PY3 = sys.version_info > (3, 0)
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
def tempdir():
import tempfile
@@ -44,13 +47,36 @@ class NoContextTests(TestCase):
encoded2 = ldb.binary_encode('test\\x')
self.assertEqual(encoded2, encoded)
-class SimpleLdb(TestCase):
+
+class LdbBaseTest(TestCase):
+ def setUp(self):
+ super(LdbBaseTest, self).setUp()
+ try:
+ if self.prefix is None:
+ self.prefix = TDB_PREFIX
+ except AttributeError:
+ self.prefix = TDB_PREFIX
+
+ def tearDown(self):
+ super(LdbBaseTest, self).tearDown()
+
+ def url(self):
+ return self.prefix + self.filename
+
+ def flags(self):
+ if self.prefix == MDB_PREFIX:
+ return ldb.FLG_NOSYNC
+ else:
+ return 0
+
+
+class SimpleLdb(LdbBaseTest):
def setUp(self):
super(SimpleLdb, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
- self.ldb = ldb.Ldb(self.filename)
+ self.ldb = ldb.Ldb(self.url(), flags=self.flags())
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -58,16 +84,15 @@ class SimpleLdb(TestCase):
# Ensure the LDB is closed now, so we close the FD
del(self.ldb)
-
def test_connect(self):
- ldb.Ldb(self.filename)
+ ldb.Ldb(self.url(), flags=self.flags())
def test_connect_none(self):
ldb.Ldb()
def test_connect_later(self):
x = ldb.Ldb()
- x.connect(self.filename)
+ x.connect(self.url(), flags=self.flags())
def test_repr(self):
x = ldb.Ldb()
@@ -82,7 +107,7 @@ class SimpleLdb(TestCase):
self.assertEqual([], x.modules())
def test_modules_tdb(self):
- x = ldb.Ldb(self.filename)
+ x = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual("[<ldb module 'tdb'>]", repr(x.modules()))
def test_firstmodule_none(self):
@@ -90,53 +115,53 @@ class SimpleLdb(TestCase):
self.assertEqual(x.firstmodule, None)
def test_firstmodule_tdb(self):
- x = ldb.Ldb(self.filename)
+ x = ldb.Ldb(self.url(), flags=self.flags())
mod = x.firstmodule
self.assertEqual(repr(mod), "<ldb module 'tdb'>")
def test_search(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search()), 0)
def test_search_controls(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(controls=["paged_results:0:5"])), 0)
def test_search_attrs(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, ""), ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
def test_search_string_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search("", ldb.SCOPE_SUBTREE, "(dc=*)", ["dc"])), 0)
def test_search_attr_string(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(TypeError, l.search, attrs="dc")
self.assertRaises(TypeError, l.search, attrs=b"dc")
def test_opaque(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.set_opaque("my_opaque", l)
self.assertTrue(l.get_opaque("my_opaque") is not None)
self.assertEqual(None, l.get_opaque("unknown"))
def test_search_scope_base_empty_db(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
ldb.SCOPE_BASE)), 0)
def test_search_scope_onelevel_empty_db(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(len(l.search(ldb.Dn(l, "dc=foo1"),
ldb.SCOPE_ONELEVEL)), 0)
def test_delete(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError, lambda: l.delete(ldb.Dn(l, "dc=foo2")))
def test_delete_w_unhandled_ctrl(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo1")
m["b"] = [b"a"]
@@ -145,10 +170,10 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_contains(self):
- name = self.filename
- l = ldb.Ldb(name)
+ name = self.url()
+ l = ldb.Ldb(name, flags=self.flags())
self.assertFalse(ldb.Dn(l, "dc=foo3") in l)
- l = ldb.Ldb(name)
+ l = ldb.Ldb(name, flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo3")
m["b"] = ["a"]
@@ -160,23 +185,23 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_get_config_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_config_basedn())
def test_get_root_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_root_basedn())
def test_get_schema_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_schema_basedn())
def test_get_default_basedn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(None, l.get_default_basedn())
def test_add(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
@@ -188,7 +213,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo4"))
def test_search_iterator(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
s = l.search_iterator()
s.abandon()
try:
@@ -288,7 +313,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = "bla"
@@ -300,7 +325,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo4"))
def test_add_w_unhandled_ctrl(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
@@ -308,7 +333,7 @@ class SimpleLdb(TestCase):
self.assertRaises(ldb.LdbError, lambda: l.add(m,["search_options:1:2"]))
def test_add_dict(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
"bla": b"bla"}
self.assertEqual(len(l.search()), 0)
@@ -319,7 +344,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_dict_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
"bla": "bla"}
self.assertEqual(len(l.search()), 0)
@@ -330,7 +355,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo5"))
def test_add_dict_string_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": "dc=foo6", "bla": b"bla"}
self.assertEqual(len(l.search()), 0)
l.add(m)
@@ -340,7 +365,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo6"))
def test_add_dict_bytes_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": b"dc=foo6", "bla": b"bla"}
self.assertEqual(len(l.search()), 0)
l.add(m)
@@ -350,7 +375,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=foo6"))
def test_rename(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo7")
m["bla"] = b"bla"
@@ -363,7 +388,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=bar"))
def test_rename_string_dns(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo8")
m["bla"] = b"bla"
@@ -377,7 +402,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=bar"))
def test_empty_dn(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertEqual(0, len(l.search()))
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=empty")
@@ -394,7 +419,7 @@ class SimpleLdb(TestCase):
self.assertEqual(0, len(rm[0]))
def test_modify_delete(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m["bla"] = [b"1234"]
@@ -417,7 +442,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modifydelete"))
def test_modify_delete_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m.text["bla"] = ["1234"]
@@ -440,7 +465,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modifydelete"))
def test_modify_add(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
@@ -458,7 +483,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_add_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
@@ -476,7 +501,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_replace(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m["bla"] = [b"1234", b"456"]
@@ -496,7 +521,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modify2"))
def test_modify_replace_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m.text["bla"] = ["1234", "456"]
@@ -516,7 +541,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=modify2"))
def test_modify_flags_change(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
@@ -542,7 +567,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_modify_flags_change_text(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
@@ -568,7 +593,7 @@ class SimpleLdb(TestCase):
l.delete(ldb.Dn(l, "dc=add"))
def test_transaction_commit(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo9"))
m["foo"] = [b"bar"]
@@ -577,7 +602,7 @@ class SimpleLdb(TestCase):
l.delete(m.dn)
def test_transaction_cancel(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo10"))
m["foo"] = [b"bar"]
@@ -588,12 +613,12 @@ class SimpleLdb(TestCase):
def test_set_debug(self):
def my_report_fn(level, text):
pass
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.set_debug(my_report_fn)
def test_zero_byte_string(self):
"""Testing we do not get trapped in the \0 byte in a property string."""
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
l.add({
"dn" : b"dc=somedn",
"objectclass" : b"user",
@@ -605,10 +630,20 @@ class SimpleLdb(TestCase):
self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
def test_no_crash_broken_expr(self):
- l = ldb.Ldb(self.filename)
+ l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
-class SearchTests(TestCase):
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SimpleLdbLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SimpleLdbLmdb, self).tearDown()
+
+class SearchTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
super(SearchTests, self).tearDown()
@@ -621,7 +656,9 @@ class SearchTests(TestCase):
super(SearchTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "search_test.ldb")
- self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l = ldb.Ldb(self.url(),
+ flags=self.flags(),
+ options=["modules:rdn_name"])
self.l.add({"dn": "@ATTRIBUTES",
"DC": "CASE_INSENSITIVE"})
@@ -1030,6 +1067,16 @@ class SearchTests(TestCase):
self.assertEqual(len(res11), 1)
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(SearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SearchTestsLmdb, self).tearDown()
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
@@ -1091,6 +1138,7 @@ class GUIDIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+
class GUIDIndexedDNFilterSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1125,8 +1173,37 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
-class AddModifyTests(TestCase):
+class AddModifyTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
super(AddModifyTests, self).tearDown()
@@ -1138,7 +1215,9 @@ class AddModifyTests(TestCase):
super(AddModifyTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "add_test.ldb")
- self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"])
+ self.l = ldb.Ldb(self.url(),
+ flags=self.flags(),
+ options=["modules:rdn_name"])
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
@@ -1266,6 +1345,16 @@ class AddModifyTests(TestCase):
"x": "z", "y": "a",
"objectUUID": b"0123456789abcde3"})
+
+class AddModifyTestsLmdb(AddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(AddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(AddModifyTestsLmdb, self).tearDown()
+
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1378,6 +1467,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
super(TransIndexedAddModifyTests, self).tearDown()
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
class DnTests(TestCase):
@@ -1989,13 +2095,13 @@ class ModuleTests(TestCase):
l = ldb.Ldb(self.filename)
self.assertEqual(["init"], ops)
-class LdbResultTests(TestCase):
+class LdbResultTests(LdbBaseTest):
def setUp(self):
super(LdbResultTests, self).setUp()
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
- self.l = ldb.Ldb(self.filename)
+ self.l = ldb.Ldb(self.url(), flags=self.flags())
self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
@@ -2103,7 +2209,7 @@ class LdbResultTests(TestCase):
del(self.l)
gc.collect()
- child_ldb = ldb.Ldb(self.filename)
+ child_ldb = ldb.Ldb(self.url(), flags=self.flags())
# start a transaction
child_ldb.transaction_start()
@@ -2174,7 +2280,7 @@ class LdbResultTests(TestCase):
del(self.l)
gc.collect()
- child_ldb = ldb.Ldb(self.filename)
+ child_ldb = ldb.Ldb(self.url(), flags=self.flags())
# start a transaction
child_ldb.transaction_start()
@@ -2240,6 +2346,16 @@ class LdbResultTests(TestCase):
self.assertEqual(got_pid, pid)
+class LdbResultTestsLmdb(LdbResultTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(LdbResultTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(LdbResultTestsLmdb, self).tearDown()
+
+
class BadTypeTests(TestCase):
def test_control(self):
l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index f7d55f6a347..591618bead3 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -32,6 +32,9 @@ import shutil
PY3 = sys.version_info > (3, 0)
+TDB_PREFIX = "tdb://"
+MDB_PREFIX = "mdb://"
+
def tempdir():
import tempfile
@@ -52,7 +55,29 @@ def contains(result, dn):
return False
-class MaxIndexKeyLengthTests(TestCase):
+class LdbBaseTest(TestCase):
+ def setUp(self):
+ super(LdbBaseTest, self).setUp()
+ try:
+ if self.prefix is None:
+ self.prefix = TDB_PREFIX
+ except AttributeError:
+ self.prefix = TDB_PREFIX
+
+ def tearDown(self):
+ super(LdbBaseTest, self).tearDown()
+
+ def url(self):
+ return self.prefix + self.filename
+
+ def flags(self):
+ if self.prefix == MDB_PREFIX:
+ return ldb.FLG_NOSYNC
+ else:
+ return 0
+
+
+class MaxIndexKeyLengthTests(LdbBaseTest):
def checkGuids(self, key, guids):
#
# This check relies on the current implementation where the indexes
@@ -89,7 +114,7 @@ class MaxIndexKeyLengthTests(TestCase):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "key_len_test.ldb")
# Note that the maximum key length is set to 50
- self.l = ldb.Ldb(self.filename,
+ self.l = ldb.Ldb(self.url(),
options=[
"modules:rdn_name",
"max_key_len_for_self_test:50"])
@@ -1217,6 +1242,17 @@ class MaxIndexKeyLengthTests(TestCase):
code = e.args[0]
self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
if __name__ == '__main__':
import unittest
unittest.TestProgram()
--
2.14.3
From e0948354a14079bf92c9f32e13f96a0af83b5c0b Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 18/20] ldb_mdb: Apply LMDB key length restrictions at
key-value layer
We need to enforce the GUID index mode so end-users do not get a supprise
in mid-operation and we enforce a max key length of 511 so that the
index key trunctation is done correctly.
Otherwise the DB will appear to work until a very long key (DN or index)
is used, after which it will be sad.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 +++++++++++++++++++++-
lib/ldb/ldb_tdb/ldb_tdb.c | 17 ++++++++++++++++-
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 24ba56112fa..1aaf6d3bfec 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -30,6 +30,8 @@
#define MDB_URL_PREFIX "mdb://"
#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
#define MEGABYTE (1024*1024)
#define GIGABYTE (1024*1024*1024)
@@ -641,7 +643,8 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
{
int ret;
unsigned int mdb_flags;
-
+ int lmdb_max_key_length;
+
if (flags & LDB_FLG_DONT_CREATE_DB) {
struct stat st;
if (stat(path, &st) != 0) {
@@ -695,6 +698,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
+ lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+ /* This will never happen, but if it does make sure to freak out */
+ if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+ talloc_free(lmdb);
+ return ldb_operr(ldb);
+ }
+
return LDB_SUCCESS;
}
@@ -746,6 +757,15 @@ int lmdb_connect(struct ldb_context *ldb,
if (flags & LDB_FLG_RDONLY) {
ltdb->read_only = true;
}
+
+ /*
+ * This maximum length becomes encoded in the index values so
+ * must never change even if LMDB starts to allow longer keys.
+ * The override option is max_key_len_for_self_test, and is
+ * used for testing only.
+ */
+ ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 206b272f28b..faf56029498 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -644,6 +644,15 @@ static int ltdb_add(struct ltdb_context *ctx)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int ret = LDB_SUCCESS;
+ if (ltdb->max_key_length != 0 &&
+ ltdb->cache->GUID_index_attribute == NULL &&
+ !ldb_dn_is_special(req->op.add.message->dn)) {
+ ldb_set_errstring(ldb_module_get_ctx(module),
+ "Must operate ldb_mdb in GUID "
+ "index mode, but " LTDB_IDXGUID " not set.");
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ret = ltdb_check_special_dn(module, req->op.add.message);
if (ret != LDB_SUCCESS) {
return ret;
@@ -2106,7 +2115,13 @@ int init_store(struct ltdb_private *ltdb,
*_module = module;
/*
- * Set the maximum key length
+ * Set or override the maximum key length
+ *
+ * The ldb_mdb code will have set this to 511, but our tests
+ * set this even smaller (to make the tests more practical).
+ *
+ * This must only be used for the selftest as the length
+ * becomes encoded in the index keys.
*/
{
const char *len_str =
--
2.14.3
From ae006b4c86159810dbc98ee66b9188e7512751b5 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:10 +1300
Subject: [PATCH 19/20] ldb: Allow tests to operate on ldb_mdb after new
restrictions
The tests were working fine on ldb_mdb but only in the mode without
the GUID index. This mode is not OK for production so has been banned
but this means we need to rework the tests to set an objectGUID on
each record.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/python/api.py | 185 +++++++++++++++++++++++++++++---------------
1 file changed, 121 insertions(+), 64 deletions(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 60678c86023..2e8d64fbba5 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -77,6 +77,10 @@ class SimpleLdb(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.ldb = ldb.Ldb(self.url(), flags=self.flags())
+ try:
+ self.ldb.add(self.index)
+ except AttributeError:
+ pass
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -165,6 +169,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo1")
m["b"] = [b"a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
self.assertRaises(ldb.LdbError, lambda: l.delete(m.dn, ["search_options:1:2"]))
l.delete(m.dn)
@@ -177,6 +182,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo3")
m["b"] = ["a"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
self.assertTrue(ldb.Dn(l, "dc=foo3") in l)
@@ -205,6 +211,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -245,6 +252,7 @@ class SimpleLdb(LdbBaseTest):
m1 = ldb.Message()
m1.dn = ldb.Dn(l, "dc=foo4")
m1["bla"] = b"bla"
+ m1["objectUUID"] = b"0123456789abcdef"
l.add(m1)
try:
s = l.search_iterator()
@@ -261,6 +269,7 @@ class SimpleLdb(LdbBaseTest):
m2 = ldb.Message()
m2.dn = ldb.Dn(l, "dc=foo5")
m2["bla"] = b"bla"
+ m2["objectUUID"] = b"0123456789abcdee"
l.add(m2)
s = l.search_iterator()
@@ -317,6 +326,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo4")
m["bla"] = "bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -335,7 +345,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": b"bla"}
+ "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -346,7 +357,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_text(self):
l = ldb.Ldb(self.url(), flags=self.flags())
m = {"dn": ldb.Dn(l, "dc=foo5"),
- "bla": "bla"}
+ "bla": "bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -356,7 +368,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_string_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": "dc=foo6", "bla": b"bla"}
+ m = {"dn": "dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -366,7 +379,8 @@ class SimpleLdb(LdbBaseTest):
def test_add_dict_bytes_dn(self):
l = ldb.Ldb(self.url(), flags=self.flags())
- m = {"dn": b"dc=foo6", "bla": b"bla"}
+ m = {"dn": b"dc=foo6", "bla": b"bla",
+ "objectUUID": b"0123456789abcdef"}
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -379,6 +393,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo7")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
try:
@@ -392,6 +407,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=foo8")
m["bla"] = b"bla"
+ m["objectUUID"] = b"0123456789abcdef"
self.assertEqual(len(l.search()), 0)
l.add(m)
self.assertEqual(len(l.search()), 1)
@@ -406,14 +422,17 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(0, len(l.search()))
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=empty")
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search()
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["blah"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -423,6 +442,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual([b"1234"], list(rm["bla"]))
@@ -434,7 +454,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -446,6 +467,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modifydelete")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
rm = l.search(m.dn)[0]
self.assertEqual(["1234"], list(rm.text["bla"]))
@@ -457,7 +479,8 @@ class SimpleLdb(LdbBaseTest):
l.modify(m)
rm = l.search(m.dn)
self.assertEqual(1, len(rm))
- self.assertEqual(set(["dn", "distinguishedName"]), set(rm[0].keys()))
+ self.assertEqual(set(["dn", "distinguishedName", "objectUUID"]),
+ set(rm[0].keys()))
rm = l.search(m.dn, attrs=["bla"])
self.assertEqual(1, len(rm))
self.assertEqual(0, len(rm[0]))
@@ -469,6 +492,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -477,7 +501,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -487,6 +511,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -495,7 +520,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
finally:
l.delete(ldb.Dn(l, "dc=add"))
@@ -505,6 +530,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m["bla"] = [b"1234", b"456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -513,7 +539,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"789"], list(rm["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -525,6 +551,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=modify2")
m.text["bla"] = ["1234", "456"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -533,7 +560,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_REPLACE, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["789"], list(rm.text["bla"]))
rm = l.search(m.dn, attrs=["bla"])[0]
self.assertEqual(1, len(rm))
@@ -545,6 +572,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m["bla"] = [b"1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -553,7 +581,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual([b"1234", b"456"], list(rm["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -571,6 +599,7 @@ class SimpleLdb(LdbBaseTest):
m = ldb.Message()
m.dn = ldb.Dn(l, "dc=add")
m.text["bla"] = ["1234"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
try:
m = ldb.Message()
@@ -579,7 +608,7 @@ class SimpleLdb(LdbBaseTest):
self.assertEqual(ldb.FLAG_MOD_ADD, m["bla"].flags())
l.modify(m)
rm = l.search(m.dn)[0]
- self.assertEqual(2, len(rm))
+ self.assertEqual(3, len(rm))
self.assertEqual(["1234", "456"], list(rm.text["bla"]))
# Now create another modify, but switch the flags before we do it
@@ -597,6 +626,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo9"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdef"
l.add(m)
l.transaction_commit()
l.delete(m.dn)
@@ -606,6 +636,7 @@ class SimpleLdb(LdbBaseTest):
l.transaction_start()
m = ldb.Message(ldb.Dn(l, "dc=foo10"))
m["foo"] = [b"bar"]
+ m["objectUUID"] = b"0123456789abcdee"
l.add(m)
l.transaction_cancel()
self.assertEqual(0, len(l.search(ldb.Dn(l, "dc=foo10"))))
@@ -625,6 +656,7 @@ class SimpleLdb(LdbBaseTest):
"cN" : b"LDAPtestUSER",
"givenname" : b"ldap",
"displayname" : b"foo\0bar",
+ "objectUUID" : b"0123456789abcdef"
})
res = l.search(expression="(dn=dc=somedn)")
self.assertEqual(b"foo\0bar", res[0]["displayname"][0])
@@ -638,6 +670,10 @@ class SimpleLdbLmdb(SimpleLdb):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SimpleLdbLmdb, self).setUp()
def tearDown(self):
@@ -659,6 +695,10 @@ class SearchTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
self.l.add({"dn": "@ATTRIBUTES",
"DC": "CASE_INSENSITIVE"})
@@ -670,7 +710,7 @@ class SearchTests(LdbBaseTest):
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
- "objectUUID": b"0123456789abcddf"})
+ "objectUUID": b"0123456789abcdef"})
self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG",
"name": b"Admins",
"x": "z", "y": "a",
@@ -1072,6 +1112,9 @@ class SearchTestsLmdb(SearchTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(SearchTestsLmdb, self).setUp()
def tearDown(self):
@@ -1129,12 +1172,12 @@ class GUIDIndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedSearchTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDXGUID = True
self.IDXONE = True
@@ -1143,15 +1186,14 @@ class GUIDIndexedDNFilterSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedDNFilterSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
@@ -1159,16 +1201,14 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
"""Test searches using the index including @IDXONE, to ensure
the index doesn't break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDAndOneLevelIndexedSearchTests, self).setUp()
self.l.add({"dn": "@OPTIONS",
"disallowDNFilter": "TRUE"})
self.disallowDNFilter = True
-
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]})
self.IDX = True
self.IDXGUID = True
self.IDXONE = True
@@ -1218,6 +1258,11 @@ class AddModifyTests(LdbBaseTest):
self.l = ldb.Ldb(self.url(),
flags=self.flags(),
options=["modules:rdn_name"])
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+
self.l.add({"dn": "DC=SAMBA,DC=ORG",
"name": b"samba.org",
"objectUUID": b"0123456789abcdef"})
@@ -1346,23 +1391,15 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
-class AddModifyTestsLmdb(AddModifyTests):
-
- def setUp(self):
- self.prefix = MDB_PREFIX
- super(AddModifyTestsLmdb, self).setUp()
-
- def tearDown(self):
- super(AddModifyTestsLmdb, self).tearDown()
-
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ if not hasattr(self, 'index'):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
+ "@IDXONE": [b"1"]}
super(IndexedAddModifyTests, self).setUp()
- self.l.add({"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou", b"objectUUID"],
- "@IDXONE": [b"1"]})
def test_duplicate_GUID(self):
try:
@@ -1436,14 +1473,12 @@ class GUIDIndexedAddModifyTests(IndexedAddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
def setUp(self):
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXATTR": [b"x", b"y", b"ou"],
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(GUIDIndexedAddModifyTests, self).setUp()
- indexlist = {"dn": "@INDEXLIST",
- "@IDXATTR": [b"x", b"y", b"ou"],
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
- m = ldb.Message.from_dict(self.l, indexlist, ldb.FLAG_MOD_REPLACE)
- self.l.modify(m)
class GUIDTransIndexedAddModifyTests(GUIDIndexedAddModifyTests):
@@ -2102,19 +2137,36 @@ class LdbResultTests(LdbBaseTest):
self.testdir = tempdir()
self.filename = os.path.join(self.testdir, "test.ldb")
self.l = ldb.Ldb(self.url(), flags=self.flags())
- self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"})
- self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins"})
- self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users"})
- self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1"})
- self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2"})
- self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3"})
- self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4"})
- self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5"})
- self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6"})
- self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7"})
- self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8"})
- self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9"})
- self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10"})
+ try:
+ self.l.add(self.index)
+ except AttributeError:
+ pass
+ self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org",
+ "objectUUID": b"0123456789abcde0"})
+ self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", "name": b"Admins",
+ "objectUUID": b"0123456789abcde1"})
+ self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", "name": b"Users",
+ "objectUUID": b"0123456789abcde2"})
+ self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", "name": b"OU #1",
+ "objectUUID": b"0123456789abcde3"})
+ self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", "name": b"OU #2",
+ "objectUUID": b"0123456789abcde4"})
+ self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", "name": b"OU #3",
+ "objectUUID": b"0123456789abcde5"})
+ self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", "name": b"OU #4",
+ "objectUUID": b"0123456789abcde6"})
+ self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", "name": b"OU #5",
+ "objectUUID": b"0123456789abcde7"})
+ self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", "name": b"OU #6",
+ "objectUUID": b"0123456789abcde8"})
+ self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", "name": b"OU #7",
+ "objectUUID": b"0123456789abcde9"})
+ self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", "name": b"OU #8",
+ "objectUUID": b"0123456789abcdea"})
+ self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", "name": b"OU #9",
+ "objectUUID": b"0123456789abcdeb"})
+ self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", "name": b"OU #10",
+ "objectUUID": b"0123456789abcdec"})
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -2215,7 +2267,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2286,7 +2339,8 @@ class LdbResultTests(LdbBaseTest):
# write to it
child_ldb.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG",
- "name": b"samba.org"})
+ "name": b"samba.org",
+ "objectUUID": b"o123456789acbdef"})
os.write(w1, b"added")
@@ -2350,6 +2404,9 @@ class LdbResultTestsLmdb(LdbResultTests):
def setUp(self):
self.prefix = MDB_PREFIX
+ self.index = {"dn": "@INDEXLIST",
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]}
super(LdbResultTestsLmdb, self).setUp()
def tearDown(self):
--
2.14.3
From 924081eedb612d20b35ec8cdb04377dc5fc1130a Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:30:43 +1300
Subject: [PATCH 20/20] ldb: Remove python warning in tests/python/index.py
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/tests/python/index.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 591618bead3..72c5c437556 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -1030,7 +1030,7 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
res = self.l.search(
base="DC=SAMBA,DC=ORG",
expression="(notUnique=" + aa_gt_max.decode("ascii") + ")")
- self.assertEquals(2, len(res))
+ self.assertEqual(2, len(res))
self.assertTrue(
contains(res, "OU=01,OU=MODIFY_NON_UNIQUE,DC=SAMBA,DC=ORG"))
self.assertTrue(
--
2.14.3
More information about the samba-technical
mailing list