[PATCH] LMDB full patch set
Andrew Bartlett
abartlet at samba.org
Mon May 7 03:02:47 UTC 2018
On Fri, 2018-05-04 at 22:42 +1200, Andrew Bartlett wrote:
> On Fri, 2018-05-04 at 06:52 +1200, Andrew Bartlett via samba-technical
> wrote:
> > On Thu, 2018-05-03 at 16:46 +0200, Stefan Metzmacher wrote:
> > >
> > > I'll try to have another look when you give me the pointers to
> > > rebased branches.
> >
> > Thanks. My plan for today (now that I'm past the flapping test) is to
> > do exactly that, there just wasn't time once I got the LSA thing done.
>
> I've finally got a set of patches I'm happy with here:
>
> https://gitlab.com/catalyst-samba/samba/commits/metze-master4-lmdb-full
These have passed a full autobuild in the gitlab CI here:
https://gitlab.com/catalyst-samba/samba/pipelines/21509189
(for the previous set) and here:
https://gitlab.com/catalyst-samba/samba/pipelines/21600415
(for the current set)
The diff against a (very rough) rebase of the previous patches I posted
is attached, just to highlight the areas of change. (That is, don't
worry about the 'wrong' stuff in the ldb_ldb stuff, it is right in the
actual patches).
Every patch passes a full make test in lib/ldb.
Gary/Garming,
Can you confirm you are happy with the review tags and review my extra
tests?
Metze,
I've addressed your concerns as far as practical[1]. If you could
please allow this to proceed into master I would most appreciate it.
Thanks,
Andrew Bartlett
[1] Moving some of the tests to ldb_lmdb_test (in a TODO commit) didn't
compile and was not practical to fix due to dependencies, so I've left
those in ldb_mod_op_test.
--
Andrew Bartlett
https://samba.org/~abartlet/
Authentication Developer, Samba Team https://samba.org
Samba Development and Support, Catalyst IT
https://catalyst.net.nz/services/samba
-------------- next part --------------
From fdf30da74daea3f95eae8333ee42707b1d5732a8 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Wed, 11 Jan 2017 17:10:19 +1300
Subject: [PATCH 01/24] ldb_mdb: Implement the lmdb backend for ldb
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 734 +++++++++++++++++++++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 58 ++++
lib/ldb/ldb_mdb/ldb_mdb_init.c | 31 ++
lib/ldb/ldb_tdb/ldb_tdb.h | 1 +
lib/ldb/tools/ldbdump.c | 126 ++++++-
lib/ldb/wscript | 99 +++++-
6 files changed, 1039 insertions(+), 10 deletions(-)
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.c
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb.h
create mode 100644 lib/ldb/ldb_mdb/ldb_mdb_init.c
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
new file mode 100644
index 00000000000..e11b44caa7b
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -0,0 +1,734 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "include/dlinklist.h"
+
+#define MDB_URL_PREFIX "mdb://"
+#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+
+#define GIGABYTE (1024*1024*1024)
+
+int ldb_mdb_err_map(int lmdb_err)
+{
+ switch (lmdb_err) {
+ case MDB_SUCCESS:
+ return LDB_SUCCESS;
+ case EIO:
+ return LDB_ERR_OPERATIONS_ERROR;
+ case MDB_INCOMPATIBLE:
+ case MDB_CORRUPTED:
+ case MDB_INVALID:
+ return LDB_ERR_UNAVAILABLE;
+ case MDB_BAD_TXN:
+ case MDB_BAD_VALSIZE:
+#ifdef MDB_BAD_DBI
+ case MDB_BAD_DBI:
+#endif
+ case MDB_PANIC:
+ case EINVAL:
+ return LDB_ERR_PROTOCOL_ERROR;
+ case MDB_MAP_FULL:
+ case MDB_DBS_FULL:
+ case MDB_READERS_FULL:
+ case MDB_TLS_FULL:
+ case MDB_TXN_FULL:
+ case EAGAIN:
+ return LDB_ERR_BUSY;
+ case MDB_KEYEXIST:
+ return LDB_ERR_ENTRY_ALREADY_EXISTS;
+ case MDB_NOTFOUND:
+ case ENOENT:
+ return LDB_ERR_NO_SUCH_OBJECT;
+ case EACCES:
+ return LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS;
+ default:
+ break;
+ }
+ return LDB_ERR_OTHER;
+}
+
+#define ldb_mdb_error(ldb, ecode) lmdb_error_at(ldb, ecode, __FILE__, __LINE__)
+static int lmdb_error_at(struct ldb_context *ldb,
+ int ecode,
+ const char *file,
+ int line)
+{
+ int ldb_err = ldb_mdb_err_map(ecode);
+ char *reason = mdb_strerror(ecode);
+ ldb_asprintf_errstring(ldb,
+ "(%d) - %s at %s:%d",
+ ecode,
+ reason,
+ file,
+ line);
+ return ldb_err;
+}
+
+
+static bool lmdb_transaction_active(struct ltdb_private *ltdb)
+{
+ return ltdb->lmdb_private->txlist != NULL;
+}
+
+static MDB_txn *lmdb_trans_get_tx(struct lmdb_trans *ltx)
+{
+ if (ltx == NULL) {
+ return NULL;
+ }
+
+ return ltx->tx;
+}
+
+static void trans_push(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ if (lmdb->txlist) {
+ talloc_steal(lmdb->txlist, ltx);
+ }
+
+ DLIST_ADD(lmdb->txlist, ltx);
+}
+
+static void trans_finished(struct lmdb_private *lmdb, struct lmdb_trans *ltx)
+{
+ DLIST_REMOVE(lmdb->txlist, ltx);
+ talloc_free(ltx);
+}
+
+
+static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx;
+
+ ltx = lmdb->txlist;
+ return ltx;
+}
+
+static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
+{
+ MDB_txn *txn = NULL;
+
+ if (lmdb->read_txn != NULL) {
+ return lmdb->read_txn;
+ }
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ int ret;
+ ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ lmdb->error = ret;
+ ldb_asprintf_errstring(lmdb->ldb,
+ "%s failed: %s\n", __FUNCTION__,
+ mdb_strerror(ret));
+ return NULL;
+ }
+ lmdb->read_txn = txn;
+ }
+
+ return txn;
+}
+
+static int lmdb_store(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val data, int flags)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ int mdb_flags;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ mdb_data.mv_size = data.length;
+ mdb_data.mv_data = data.data;
+
+ if (flags == TDB_INSERT) {
+ mdb_flags = MDB_NOOVERWRITE;
+ } else if ((flags == TDB_MODIFY)) {
+ /*
+ * Modifying a record, ensure that it exists.
+ * This mimics the TDB semantics
+ */
+ MDB_val value;
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
+ if (lmdb->error != MDB_SUCCESS) {
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ mdb_flags = 0;
+ } else {
+ mdb_flags = 0;
+ }
+
+ lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_traverse_fn(struct ltdb_private *ltdb,
+ ldb_kv_traverse_fn fn,
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi = 0;
+ MDB_cursor *cursor = NULL;
+ int ret;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_cursor_open(txn, dbi, &cursor);
+ if (lmdb->error != MDB_SUCCESS) {
+ goto done;
+ }
+
+ while ((lmdb->error = mdb_cursor_get(
+ cursor, &mdb_key,
+ &mdb_data, MDB_NEXT)) == MDB_SUCCESS) {
+
+ struct ldb_val key = {
+ .length = mdb_key.mv_size,
+ .data = mdb_key.mv_data,
+ };
+ struct ldb_val data = {
+ .length = mdb_data.mv_size,
+ .data = mdb_data.mv_data,
+ };
+
+ ret = fn(ltdb, key, data, ctx);
+ if (ret != 0) {
+ goto done;
+ }
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ lmdb->error = MDB_SUCCESS;
+ }
+done:
+ if (cursor != NULL) {
+ mdb_cursor_close(cursor);
+ }
+
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
+ struct ldb_val key,
+ struct ldb_val key2,
+ struct ldb_val data,
+ void *state)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct ldb_val copy;
+ int ret = LDB_SUCCESS;
+
+ /*
+ * Need to take a copy of the data as the delete operation alters the
+ * data, as it is in private lmdb memory.
+ */
+ copy.length = data.length;
+ copy.data = talloc_memdup(ltdb, data.data, data.length);
+ if (copy.data == NULL) {
+ lmdb->error = MDB_PANIC;
+ return ldb_oom(lmdb->ldb);
+ }
+
+ lmdb->error = lmdb_delete(ltdb, key);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to delete %*.*s "
+ "for rekey as %*.*s: %s",
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+
+ lmdb->error = lmdb_store(ltdb, key2, copy, 0);
+ if (lmdb->error != MDB_SUCCESS) {
+ ldb_debug(
+ lmdb->ldb,
+ LDB_DEBUG_ERROR,
+ "Failed to rekey %*.*s as %*.*s: %s",
+ (int)key.length, (int)key.length,
+ (const char *)key.data,
+ (int)key2.length, (int)key2.length,
+ (const char *)key.data,
+ mdb_strerror(lmdb->error));
+ ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
+ goto done;
+ }
+
+done:
+ if (copy.data != NULL) {
+ TALLOC_FREE(copy.data);
+ copy.length = 0;
+ }
+
+ /*
+ * Explicity invalidate the data, as the delete has done this
+ */
+ data.length = 0;
+ data.data = NULL;
+
+ return ret;
+}
+
+/* Handles only a single record */
+static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
+ int (*parser)(struct ldb_val key, struct ldb_val data,
+ void *private_data),
+ void *ctx)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ MDB_val mdb_key;
+ MDB_val mdb_data;
+ MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct ldb_val data;
+
+ txn = get_current_txn(lmdb);
+ if (txn == NULL) {
+ ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction active");
+ lmdb->error = MDB_PANIC;
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ lmdb->error = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ mdb_key.mv_size = key.length;
+ mdb_key.mv_data = key.data;
+
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+ if (lmdb->error != MDB_SUCCESS) {
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ if (lmdb->error == MDB_NOTFOUND) {
+ return LDB_ERR_NO_SUCH_OBJECT;
+ }
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+ data.data = mdb_data.mv_data;
+ data.length = mdb_data.mv_size;
+
+ /* TODO closing a handle should not even be necessary */
+ mdb_dbi_close(lmdb->env, dbi);
+
+ /* We created a read transaction, commit it */
+ if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ }
+ return parser(key, data, ctx);
+}
+
+
+static int lmdb_lock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ lmdb->error = MDB_SUCCESS;
+ if (lmdb_transaction_active(ltdb) == false &&
+ ltdb->read_lock_count == 0) {
+ lmdb->error = mdb_txn_begin(lmdb->env,
+ NULL,
+ MDB_RDONLY,
+ &lmdb->read_txn);
+ }
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ ltdb->read_lock_count++;
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_unlock_read(struct ldb_module *module)
+{
+ void *data = ldb_module_get_private(module);
+ struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+
+ if (lmdb_transaction_active(ltdb) == false && ltdb->read_lock_count == 1) {
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ mdb_txn_commit(lmdb->read_txn);
+ lmdb->read_txn = NULL;
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+ }
+ ltdb->read_lock_count--;
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_start(struct ltdb_private *ltdb)
+{
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+ struct lmdb_trans *ltx;
+ struct lmdb_trans *ltx_head;
+ MDB_txn *tx_parent;
+
+ ltx = talloc_zero(lmdb, struct lmdb_trans);
+ if (ltx == NULL) {
+ return ldb_oom(lmdb->ldb);
+ }
+
+ ltx_head = lmdb_private_trans_head(lmdb);
+
+ tx_parent = lmdb_trans_get_tx(ltx_head);
+
+ lmdb->error = mdb_txn_begin(lmdb->env, tx_parent, 0, <x->tx);
+ if (lmdb->error != MDB_SUCCESS) {
+ return ldb_mdb_error(lmdb->ldb, lmdb->error);
+ }
+
+ trans_push(lmdb, ltx);
+
+ return ldb_mdb_err_map(lmdb->error);
+}
+
+static int lmdb_transaction_cancel(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_prepare_commit(struct ltdb_private *ltdb)
+{
+ /* No need to prepare a commit */
+ return LDB_SUCCESS;
+}
+
+static int lmdb_transaction_commit(struct ltdb_private *ltdb)
+{
+ struct lmdb_trans *ltx;
+ struct lmdb_private *lmdb = ltdb->lmdb_private;
+
+ ltx = lmdb_private_trans_head(lmdb);
+ if (ltx == NULL) {
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ lmdb->error = mdb_txn_commit(ltx->tx);
+ trans_finished(lmdb, ltx);
+
+ return lmdb->error;
+}
+
+static int lmdb_error(struct ltdb_private *ltdb)
+{
+ return ldb_mdb_err_map(ltdb->lmdb_private->error);
+}
+
+static const char *lmdb_errorstr(struct ltdb_private *ltdb)
+{
+ return mdb_strerror(ltdb->lmdb_private->error);
+}
+
+static const char * lmdb_name(struct ltdb_private *ltdb)
+{
+ return "lmdb";
+}
+
+static bool lmdb_changed(struct ltdb_private *ltdb)
+{
+ /*
+ * lmdb does no provide a quick way to determine if the database
+ * has changed. This function always returns true.
+ *
+ * Note that tdb uses a sequence number that allows this function
+ * to be implemented efficiently.
+ */
+ return true;
+}
+
+static struct kv_db_ops lmdb_key_value_ops = {
+ .store = lmdb_store,
+ .delete = lmdb_delete,
+ .iterate = lmdb_traverse_fn,
+ .update_in_iterate = lmdb_update_in_iterate,
+ .fetch_and_parse = lmdb_parse_record,
+ .lock_read = lmdb_lock_read,
+ .unlock_read = lmdb_unlock_read,
+ .begin_write = lmdb_transaction_start,
+ .prepare_write = lmdb_transaction_prepare_commit,
+ .finish_write = lmdb_transaction_commit,
+ .abort_write = lmdb_transaction_cancel,
+ .error = lmdb_error,
+ .errorstr = lmdb_errorstr,
+ .name = lmdb_name,
+ .has_changed = lmdb_changed,
+ .transaction_active = lmdb_transaction_active,
+};
+
+static const char *lmdb_get_path(const char *url)
+{
+ const char *path;
+
+ /* parse the url */
+ if (strchr(url, ':')) {
+ if (strncmp(url, MDB_URL_PREFIX, MDB_URL_PREFIX_SIZE) != 0) {
+ return NULL;
+ }
+ path = url + MDB_URL_PREFIX_SIZE;
+ } else {
+ path = url;
+ }
+
+ return path;
+}
+
+static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
+{
+ struct lmdb_trans *ltx = NULL;
+
+ /*
+ * Close the read transaction if it's open
+ */
+ if (lmdb->read_txn != NULL) {
+ mdb_txn_abort(lmdb->read_txn);
+ }
+
+ if (lmdb->env == NULL) {
+ return 0;
+ }
+
+ /*
+ * Abort any currently active transactions
+ */
+ ltx = lmdb_private_trans_head(lmdb);
+ while (ltx != NULL) {
+ mdb_txn_abort(ltx->tx);
+ trans_finished(lmdb, ltx);
+ ltx = lmdb_private_trans_head(lmdb);
+ }
+
+ mdb_env_close(lmdb->env);
+ lmdb->env = NULL;
+
+ return 0;
+}
+
+static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags,
+ struct lmdb_private *lmdb)
+{
+ int ret;
+ unsigned int mdb_flags;
+
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
+ }
+
+ ret = mdb_env_create(&lmdb->env);
+ if (ret != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
+ ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ if (ret != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not open MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return ldb_mdb_err_map(ret);
+ }
+
+ mdb_env_set_maxreaders(lmdb->env, 100000);
+ /* MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
+ mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ if (flags & LDB_FLG_RDONLY) {
+ mdb_flags |= MDB_RDONLY;
+ }
+ ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+ if (ret != 0) {
+ ldb_asprintf_errstring(ldb,
+ "Could not open DB %s: %s\n",
+ path, mdb_strerror(ret));
+ talloc_free(lmdb);
+ return ldb_mdb_err_map(ret);
+ }
+
+ return LDB_SUCCESS;
+}
+
+int lmdb_connect(struct ldb_context *ldb,
+ const char *url,
+ unsigned int flags,
+ const char *options[],
+ struct ldb_module **_module)
+{
+ const char *path = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct ltdb_private *ltdb = NULL;
+ int ret;
+
+ /*
+ * We hold locks, so we must use a private event context
+ * on each returned handle
+ */
+ ldb_set_require_private_event_context(ldb);
+
+ path = lmdb_get_path(url);
+ if (path == NULL) {
+ ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid mdb URL '%s'", url);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ ltdb = talloc_zero(ldb, struct ltdb_private);
+ if (!ltdb) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ lmdb = talloc_zero(ldb, struct lmdb_private);
+ if (lmdb == NULL) {
+ TALLOC_FREE(ltdb);
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ lmdb->ldb = ldb;
+ ltdb->kv_ops = &lmdb_key_value_ops;
+
+ ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+
+ ltdb->lmdb_private = lmdb;
+ if (flags & LDB_FLG_RDONLY) {
+ ltdb->read_only = true;
+ }
+
+ return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
new file mode 100644
index 00000000000..d4a635ca693
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -0,0 +1,58 @@
+/*
+ ldb database library using mdb back end - transaction operations
+
+ Copyright (C) Jakub Hrozek 2015
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _LDB_MDB_H_
+#define _LDB_MDB_H_
+
+#include "ldb_private.h"
+#include <lmdb.h>
+
+struct lmdb_private {
+ struct ldb_context *ldb;
+ MDB_env *env;
+
+ struct lmdb_trans *txlist;
+
+ struct ldb_mdb_metadata {
+ struct ldb_message *attributes;
+ unsigned seqnum;
+ } *meta;
+ int error;
+ MDB_txn *read_txn;
+
+};
+
+struct lmdb_trans {
+ struct lmdb_trans *next;
+ struct lmdb_trans *prev;
+
+ MDB_txn *tx;
+};
+
+int ldb_mdb_err_map(int lmdb_err);
+int lmdb_connect(struct ldb_context *ldb, const char *url,
+ unsigned int flags, const char *options[],
+ struct ldb_module **_module);
+
+#endif /* _LDB_MDB_H_ */
diff --git a/lib/ldb/ldb_mdb/ldb_mdb_init.c b/lib/ldb/ldb_mdb/ldb_mdb_init.c
new file mode 100644
index 00000000000..339c3f22b2a
--- /dev/null
+++ b/lib/ldb/ldb_mdb/ldb_mdb_init.c
@@ -0,0 +1,31 @@
+/*
+ ldb database library using mdb back end
+
+ Copyright (C) Jakub Hrozek 2014
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ ** NOTE! The following LGPL license applies to the ldb
+ ** library. This does NOT imply that all of Samba is released
+ ** under the LGPL
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ldb_mdb.h"
+
+int ldb_mdb_init(const char *version)
+{
+ LDB_MODULE_CHECK_VERSION(version);
+ return ldb_register_backend("mdb", lmdb_connect, false);
+}
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 4d531208da6..9c3f8d89d8d 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -37,6 +37,7 @@ struct kv_db_ops {
struct ltdb_private {
const struct kv_db_ops *kv_ops;
TDB_CONTEXT *tdb;
+ struct lmdb_private *lmdb_private;
unsigned int connect_flags;
unsigned long long sequence_number;
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index c399b59eca4..2da2ca8ec70 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -27,6 +27,11 @@
#include <ldb.h>
#include <ldb_private.h>
+#ifdef HAVE_LMDB
+#include "lmdb.h"
+#endif /* ifdef HAVE_LMDB */
+
+
static struct ldb_context *ldb;
bool show_index = false;
bool validate_contents = false;
@@ -166,6 +171,116 @@ static int dump_tdb(const char *fname, struct ldb_dn *dn, bool emergency)
return tdb_traverse(tdb, traverse_fn, dn) == -1 ? 1 : 0;
}
+#ifdef HAVE_LMDB
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ int ret;
+ struct MDB_env *env = NULL;
+ struct MDB_txn *txn = NULL;
+ MDB_dbi dbi;
+ struct MDB_cursor *cursor = NULL;
+ struct MDB_val key;
+ struct MDB_val data;
+
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not create MDB environment: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_env_open(env,
+ fname,
+ MDB_NOSUBDIR|MDB_NOTLS|MDB_RDONLY,
+ 0600);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open environment for %s: (%d) %s\n",
+ fname,
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not start transaction: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_env;
+ }
+
+ ret = mdb_dbi_open(txn, NULL, 0, &dbi);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open database: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_open(txn, dbi, &cursor);
+ if (ret != 0) {
+ fprintf(stderr,
+ "Could not open cursor: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_txn;
+ }
+
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not find first record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ while (ret != MDB_NOTFOUND) {
+ struct TDB_DATA tkey = {
+ .dptr = key.mv_data,
+ .dsize = key.mv_size
+ };
+ struct TDB_DATA tdata = {
+ .dptr = data.mv_data,
+ .dsize = data.mv_size
+ };
+ traverse_fn(NULL, tkey, tdata, dn);
+ ret = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ if (ret != 0 && ret != MDB_NOTFOUND) {
+ fprintf(stderr,
+ "Could not read next record: (%d) %s\n",
+ ret,
+ mdb_strerror(ret));
+ goto close_cursor;
+ }
+ }
+ ret = 0;
+
+close_cursor:
+ mdb_cursor_close(cursor);
+close_txn:
+ mdb_txn_commit(txn);
+close_env:
+ mdb_env_close(env);
+
+ if (ret != 0) {
+ return 1;
+ }
+ return 0;
+
+}
+#else
+static int dump_lmdb(const char *fname, struct ldb_dn *dn, bool emergency)
+{
+ /* not built with lmdb support */
+ return 1;
+}
+#endif /* #ifdef HAVE_LMDB */
+
static void usage( void)
{
printf( "Usage: ldbdump [options] <filename>\n\n");
@@ -229,5 +344,14 @@ static void usage( void)
fname = argv[optind];
- return dump_tdb(fname, dn, emergency);
+ rc = dump_lmdb(fname, dn, emergency);
+ if (rc != 0) {
+ rc = dump_tdb(fname, dn, emergency);
+ if (rc != 0) {
+ fprintf(stderr, "Failed to open %s\n", fname);
+ return 1;
+ }
+ }
+ return 0;
+
}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 35549d528fd..ad87b136645 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -13,7 +13,7 @@ while not os.path.exists(srcdir+'/buildtools') and len(srcdir.split('/')) < 5:
srcdir = srcdir + '/..'
sys.path.insert(0, srcdir + '/buildtools/wafsamba')
-import wafsamba, samba_dist, Utils
+import wafsamba, samba_dist, Utils, Options
samba_dist.DIST_DIRS('''lib/ldb:. lib/replace:lib/replace lib/talloc:lib/talloc
lib/tdb:lib/tdb lib/tdb:lib/tdb lib/tevent:lib/tevent
@@ -92,6 +92,16 @@ def configure(conf):
implied_deps='replace talloc tdb tevent'):
conf.define('USING_SYSTEM_LDB', 1)
+ if conf.env.standalone_ldb:
+ # Require lmdb support for standalone mode.
+ conf.env.REQUIRE_LMDB = True
+ elif not Options.options.without_ad_dc:
+ # Require lmdb support for addc mode
+ conf.env.REQUIRE_LMDB = True
+ else:
+ conf.env.REQUIRE_LMDB = False
+
+
if conf.CONFIG_SET('USING_SYSTEM_LDB'):
v = VERSION.split('.')
conf.DEFINE('EXPECTED_SYSTEM_LDB_VERSION_MAJOR', int(v[0]))
@@ -110,6 +120,36 @@ def configure(conf):
if not sys.platform.startswith("openbsd"):
conf.ADD_LDFLAGS('-Wl,-no-undefined', testflags=True)
+ # if lmdb support is enabled then we require lmdb
+ # is present, build the mdb back end and enable lmdb support in
+ # the tools.
+ if conf.env.REQUIRE_LMDB:
+ if not conf.CHECK_CFG(package='lmdb',
+ args='"lmdb >= 0.9.16" --cflags --libs',
+ msg='Checking for lmdb >= 0.9.16',
+ mandatory=False):
+ if not conf.CHECK_CODE('''
+ #if MDB_VERSION_MAJOR == 0 \
+ && MDB_VERSION_MINOR <= 9 \
+ && MDB_VERSION_PATCH < 16
+ #error LMDB too old
+ #endif
+ ''',
+ 'HAVE_GOOD_LMDB_VERSION',
+ headers='lmdb.h',
+ msg='Checking for lmdb >= 0.9.16 via header check'):
+
+ if conf.env.standalone_ldb:
+ raise Utils.WafError('ldb requires '
+ 'lmdb 0.9.16 or later')
+ elif not Options.options.without_ad_dc:
+ raise Utils.WafError('Samba AD DC requires '
+ 'lmdb 0.9.16 or later')
+
+ if conf.CHECK_FUNCS_IN('mdb_env_create', 'lmdb', headers='lmdb.h'):
+ conf.DEFINE('HAVE_LMDB', '1')
+ conf.env.ENABLE_MDB_BACKEND = True
+
conf.DEFINE('HAVE_CONFIG_H', 1, add_to_cflags=True)
conf.SAMBA_CONFIG_H()
@@ -321,13 +361,33 @@ def build(bld):
private_library=True,
deps='tdb ldb')
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_MODULE('ldb_mdb',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb_init.c'''),
+ init_function='ldb_mdb_init',
+ module_init_name='ldb_init_module',
+ internal_module=False,
+ deps='ldb ldb_key_value ldb_mdb_int',
+ subsystem='ldb')
+
+ bld.SAMBA_LIBRARY('ldb_mdb_int',
+ bld.SUBDIR('ldb_mdb',
+ '''ldb_mdb.c '''),
+ private_library=True,
+ deps='ldb lmdb ldb_key_value')
+ lmdb_deps = ' ldb_mdb_int'
+ else:
+ lmdb_deps = ''
+
+
bld.SAMBA_MODULE('ldb_ldb',
bld.SUBDIR('ldb_ldb',
'''ldb_ldb.c'''),
init_function='ldb_ldb_init',
module_init_name='ldb_init_module',
internal_module=False,
- deps='ldb ldb_key_value',
+ deps='ldb ldb_key_value' + lmdb_deps,
subsystem='ldb')
# have a separate subsystem for common/ldb.c, so it can rebuild
@@ -347,8 +407,14 @@ def build(bld):
bld.SAMBA_BINARY('ldbtest', 'tools/ldbtest.c', deps='ldb-cmdline ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ lmdb_deps = ' lmdb'
+ else:
+ lmdb_deps = ''
# ldbdump doesn't get installed
- bld.SAMBA_BINARY('ldbdump', 'tools/ldbdump.c', deps='ldb-cmdline ldb',
+ bld.SAMBA_BINARY('ldbdump',
+ 'tools/ldbdump.c',
+ deps='ldb-cmdline ldb' + lmdb_deps,
install=False)
bld.SAMBA_LIBRARY('ldb-cmdline',
@@ -374,6 +440,18 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ if bld.CONFIG_SET('HAVE_LMDB'):
+ bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
+ source='tests/ldb_mod_op_test.c',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
+ deps='cmocka ldb',
+ install=False)
+ bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
+ source='tests/ldb_kv_ops_test.c',
+ cflags='-DTEST_BE=\"mdb\"',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_msg_test',
source='tests/ldb_msg.c',
deps='cmocka ldb',
@@ -411,12 +489,15 @@ def test(ctx):
print("Python testsuite returned %d" % pyret)
cmocka_ret = 0
- for test_exe in ['test_ldb_qsort',
- 'ldb_msg_test',
- 'ldb_tdb_mod_op_test',
- 'ldb_tdb_guid_mod_op_test',
- 'ldb_msg_test',
- 'ldb_tdb_kv_ops_test']:
+ test_exes = ['test_ldb_qsort',
+ 'ldb_msg_test',
+ 'ldb_tdb_mod_op_test',
+ 'ldb_tdb_guid_mod_op_test',
+ 'ldb_tdb_kv_ops_test']
+
+ if env.ENABLE_MDB_BACKEND:
+ test_exes.append('ldb_mdb_mod_op_test')
+ for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.11.0
From e239d83c78f99bbea92171a04bde329e0211b853 Mon Sep 17 00:00:00 2001
From: Stefan Metzmacher <metze at samba.org>
Date: Thu, 12 Apr 2018 12:51:39 +0200
Subject: [PATCH 02/24] lib/ldb/tools/ldbdump.c #include <lmdb.h>
---
lib/ldb/tools/ldbdump.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index 2da2ca8ec70..4697661a59d 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -28,7 +28,7 @@
#include <ldb_private.h>
#ifdef HAVE_LMDB
-#include "lmdb.h"
+#include <lmdb.h>
#endif /* ifdef HAVE_LMDB */
--
2.11.0
From d76c42182be66a3b35c4941a771394de2113455c Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 1 Mar 2018 16:53:07 +1300
Subject: [PATCH 03/24] ldb_mdb: Enable LDB_FLG_NOSYNC in ldb_mdb
This is used in selftest with 'ldb:nosync = true'.
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index e11b44caa7b..ad38849bddd 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -670,6 +670,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
if (flags & LDB_FLG_RDONLY) {
mdb_flags |= MDB_RDONLY;
}
+ if (flags & LDB_FLG_NOSYNC) {
+ mdb_flags |= MDB_NOSYNC;
+ }
ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
--
2.11.0
From c3841af6847874027e426d84ffdf92f9a5fe1a26 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Mon, 5 Mar 2018 16:04:03 +1300
Subject: [PATCH 04/24] ldb_mdb: Store pid to change destructor on fork
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 22 ++++++++++++++++++++++
lib/ldb/ldb_mdb/ldb_mdb.h | 2 ++
2 files changed, 24 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index ad38849bddd..d0619f1ea22 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -595,6 +595,25 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
{
struct lmdb_trans *ltx = NULL;
+ /* Check if this is a forked child */
+ if (getpid() != lmdb->pid) {
+ int fd = 0;
+ /*
+ * We cannot call mdb_env_close or commit any transactions,
+ * otherwise they might appear finished in the parent.
+ *
+ */
+
+ if (mdb_env_get_fd(lmdb->env, &fd) == 0) {
+ close(fd);
+ }
+
+ /* Remove the pointer, so that no access should occur */
+ lmdb->env = NULL;
+
+ return 0;
+ }
+
/*
* Close the read transaction if it's open
*/
@@ -682,6 +701,9 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
return ldb_mdb_err_map(ret);
}
+ /* Store the original pid during the LMDB open */
+ lmdb->pid = getpid();
+
return LDB_SUCCESS;
}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index d4a635ca693..8f21493927b 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -41,6 +41,8 @@ struct lmdb_private {
int error;
MDB_txn *read_txn;
+ pid_t pid;
+
};
struct lmdb_trans {
--
2.11.0
From 291cedef3e40b18ee0ccd578d3e3bb5c34b198dc Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 05/24] ldb_mdb: Don't allow modify operations on a read only
db
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Stefan Metzmacher <metze at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index d0619f1ea22..c07a144096c 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -162,6 +162,10 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -216,6 +220,10 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
MDB_txn *txn = NULL;
MDB_dbi dbi = 0;
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -472,6 +480,11 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
struct lmdb_trans *ltx_head;
MDB_txn *tx_parent;
+ /* Do not take out the transaction lock on a read-only DB */
+ if (ltdb->read_only) {
+ return LDB_ERR_UNWILLING_TO_PERFORM;
+ }
+
ltx = talloc_zero(lmdb, struct lmdb_trans);
if (ltx == NULL) {
return ldb_oom(lmdb->ldb);
--
2.11.0
From 0d94678d56325a44ad026e1054f3ff208e0ae60e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 2 Feb 2018 15:30:53 +1300
Subject: [PATCH 06/24] ldb_mdb/tests: Add tests to check for max key length
and DB size
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_size_test.c | 210 ++++++++++++++++++++++++++++++++
lib/ldb/tests/ldb_lmdb_test.c | 240 +++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 15 +++
3 files changed, 465 insertions(+)
create mode 100644 lib/ldb/tests/ldb_lmdb_size_test.c
create mode 100644 lib/ldb/tests/ldb_lmdb_test.c
diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
new file mode 100644
index 00000000000..af015fa72b5
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -0,0 +1,210 @@
+/*
+ * lmdb backend specific tests for ldb
+ * Tests for truncated index keys
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * These tests confirm that database sizes of > 4GB are supported
+ * Due to the disk space requirement they are not run as part of the normal
+ * self test runs.
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+#include <tevent.h>
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include <lmdb.h>
+
+
+#define TEST_BE "mdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_db_size_gt_4GB(void **state)
+{
+ int ret, x;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ const int MB = 1024 * 1024;
+ char *blob = NULL;
+
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+
+ blob = talloc_zero_size(tmp_ctx, (MB + 1));
+ assert_non_null(blob);
+ memset(blob, 'x', MB);
+
+
+ for (x = 0; x < 6144; x++) {
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=test%d", x);
+ assert_non_null(msg->dn);
+
+ ldb_transaction_start(test_ctx->ldb);
+ ret = ldb_msg_add_string(msg, "blob", blob);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+ ldb_transaction_commit(test_ctx->ldb);
+
+ TALLOC_FREE(msg);
+ }
+ talloc_free(tmp_ctx);
+ {
+ struct stat s;
+ ret = stat(test_ctx->dbfile, &s);
+ assert_int_equal(ret, 0);
+ assert_true(s.st_size > (6144LL * MB));
+ }
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_db_size_gt_4GB,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
new file mode 100644
index 00000000000..e47c7dbc70f
--- /dev/null
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -0,0 +1,240 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+#include <tevent.h>
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include <lmdb.h>
+
+
+#define TEST_BE "mdb"
+
+#define LMDB_MAX_KEY_SIZE 511
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+ const char *lockfile; /* lockfile is separate */
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->lockfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->lockfile = talloc_asprintf(test_ctx, "%s-lock",
+ test_ctx->dbfile);
+ assert_non_null(test_ctx->lockfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+static void test_ldb_add_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, 0);
+
+ talloc_free(tmp_ctx);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_eq_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index ad87b136645..621cfcf4b77 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -446,6 +446,17 @@ def build(bld):
cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
deps='cmocka ldb',
install=False)
+
+ bld.SAMBA_BINARY('ldb_lmdb_test',
+ source='tests/ldb_lmdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+
+ bld.SAMBA_BINARY('ldb_lmdb_size_test',
+ source='tests/ldb_lmdb_size_test.c',
+ deps='cmocka ldb',
+ install=False)
+
bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
source='tests/ldb_kv_ops_test.c',
cflags='-DTEST_BE=\"mdb\"',
@@ -497,6 +508,10 @@ def test(ctx):
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
+ test_exes.append('ldb_lmdb_test')
+ # we don't want to run ldb_lmdb_size_test (which proves we can
+ # fit > 4G of data into the DB), it would fill up the disk on
+ # many of our test instances
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.11.0
From f6a1dae510c7e24794436c68dea5d752fb9b74c6 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 09:13:31 +1300
Subject: [PATCH 07/24] ldb_mdb/tests: Run api and index test also on lmdb
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/python/api.py | 97 ++++++++++++++++++++++++++++++++++++++++++-
lib/ldb/tests/python/index.py | 11 +++++
2 files changed, 107 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index c5935ba3497..72fc0d9624e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -15,6 +15,12 @@ PY3 = sys.version_info > (3, 0)
TDB_PREFIX = "tdb://"
MDB_PREFIX = "mdb://"
+MDB_INDEX_OBJ = {
+ "dn": "@INDEXLIST",
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]
+}
def tempdir():
import tempfile
@@ -665,6 +671,17 @@ class SimpleLdb(LdbBaseTest):
l = ldb.Ldb(self.url(), flags=self.flags())
self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"]))
+# Run the SimpleLdb tests against an lmdb backend
+class SimpleLdbLmdb(SimpleLdb):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ self.index = MDB_INDEX_OBJ
+ super(SimpleLdbLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SimpleLdbLmdb, self).tearDown()
+
class SearchTests(LdbBaseTest):
def tearDown(self):
shutil.rmtree(self.testdir)
@@ -1093,6 +1110,18 @@ class SearchTests(LdbBaseTest):
self.assertEqual(len(res11), 1)
+# Run the search tests against an lmdb backend
+class SearchTestsLmdb(SearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ self.index = MDB_INDEX_OBJ
+ super(SearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(SearchTestsLmdb, self).tearDown()
+
+
class IndexedSearchTests(SearchTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1185,6 +1214,35 @@ class GUIDAndOneLevelIndexedSearchTests(SearchTests):
self.IDXGUID = True
self.IDXONE = True
+class GUIDIndexedSearchTestsLmdb(GUIDIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedSearchTestsLmdb, self).tearDown()
+
+
+class GUIDIndexedDNFilterSearchTestsLmdb(GUIDIndexedDNFilterSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDIndexedDNFilterSearchTestsLmdb, self).tearDown()
+
+
+class GUIDAndOneLevelIndexedSearchTestsLmdb(GUIDAndOneLevelIndexedSearchTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GUIDAndOneLevelIndexedSearchTestsLmdb, self).tearDown()
+
class AddModifyTests(LdbBaseTest):
def tearDown(self):
@@ -1347,6 +1405,16 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
+class AddModifyTestsLmdb(AddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ self.index = MDB_INDEX_OBJ
+ super(AddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(AddModifyTestsLmdb, self).tearDown()
+
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -1457,6 +1525,23 @@ class TransIndexedAddModifyTests(IndexedAddModifyTests):
self.l.transaction_commit()
super(TransIndexedAddModifyTests, self).tearDown()
+class GuidIndexedAddModifyTestsLmdb(GUIDIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidIndexedAddModifyTestsLmdb, self).tearDown()
+
+class GuidTransIndexedAddModifyTestsLmdb(GUIDTransIndexedAddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(GuidTransIndexedAddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(GuidTransIndexedAddModifyTestsLmdb, self).tearDown()
class BadIndexTests(LdbBaseTest):
def setUp(self):
@@ -1617,7 +1702,6 @@ class GUIDBadIndexTests(BadIndexTests):
super(GUIDBadIndexTests, self).setUp()
-
class DnTests(TestCase):
def setUp(self):
@@ -2498,6 +2582,17 @@ class LdbResultTests(LdbBaseTest):
self.assertEqual(got_pid, pid)
+class LdbResultTestsLmdb(LdbResultTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ self.index = MDB_INDEX_OBJ
+ super(LdbResultTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(LdbResultTestsLmdb, self).tearDown()
+
+
class BadTypeTests(TestCase):
def test_control(self):
l = ldb.Ldb()
diff --git a/lib/ldb/tests/python/index.py b/lib/ldb/tests/python/index.py
index 9b9e4f3469f..3379fb9374f 100755
--- a/lib/ldb/tests/python/index.py
+++ b/lib/ldb/tests/python/index.py
@@ -1280,6 +1280,17 @@ class MaxIndexKeyLengthTests(LdbBaseTest):
code = e.args[0]
self.assertEqual(ldb.ERR_NO_SUCH_OBJECT, code)
+
+# Run the index truncation tests against an lmdb backend
+class MaxIndexKeyLengthTestsLmdb(MaxIndexKeyLengthTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ super(MaxIndexKeyLengthTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(MaxIndexKeyLengthTestsLmdb, self).tearDown()
+
if __name__ == '__main__':
import unittest
unittest.TestProgram()
--
2.11.0
From 91199a57be1f55c2ae9f50cc4479da007c45790e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:27:51 +1300
Subject: [PATCH 08/24] ldb_mdb: Apply LMDB key length restrictions at
key-value layer
We need to enforce the GUID index mode so end-users do not get a
supprise in mid-operation and we enforce a max key length of 511 so
that the index key trunctation is done correctly.
Otherwise the DB will appear to work until a very long key (DN or
index) is used, after which it will be sad.
Because the previous ldb_lmdb_test confirmed the key length by
creating a large DN, those tests are re-worked to use the GUID index
mode. In turn, new tests are written that create a special DN around
the maximum key length.
Finally a test is included that demonstrates that adding entries to
the LMDB DB without GUID index mode fails.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 19 +++++
lib/ldb/tests/ldb_lmdb_test.c | 166 +++++++++++++++++++++++++++++++++++++++++-
2 files changed, 182 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index c07a144096c..139140bb621 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -29,6 +29,8 @@
#define MDB_URL_PREFIX "mdb://"
#define MDB_URL_PREFIX_SIZE (sizeof(MDB_URL_PREFIX)-1)
+#define LDB_MDB_MAX_KEY_LENGTH 511
+
#define GIGABYTE (1024*1024*1024)
int ldb_mdb_err_map(int lmdb_err)
@@ -662,6 +664,7 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
{
int ret;
unsigned int mdb_flags;
+ int lmdb_max_key_length;
if (flags & LDB_FLG_DONT_CREATE_DB) {
struct stat st;
@@ -717,6 +720,14 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
+ lmdb_max_key_length = mdb_env_get_maxkeysize(lmdb->env);
+
+ /* This will never happen, but if it does make sure to freak out */
+ if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
+ talloc_free(lmdb);
+ return ldb_operr(ldb);
+ }
+
return LDB_SUCCESS;
}
@@ -768,5 +779,13 @@ int lmdb_connect(struct ldb_context *ldb,
ltdb->read_only = true;
}
+ /*
+ * This maximum length becomes encoded in the index values so
+ * must never change even if LMDB starts to allow longer keys.
+ * The override option is max_key_len_for_self_test, and is
+ * used for testing only.
+ */
+ ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
+
return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index e47c7dbc70f..eece2be8057 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -132,12 +132,22 @@ static int ldbtest_setup(void **state)
{
struct ldbtest_ctx *test_ctx;
int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
ldbtest_noconn_setup((void **) &test_ctx);
ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
*state = test_ctx;
return 0;
}
@@ -167,7 +177,8 @@ static void test_ldb_add_key_len_gt_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -181,8 +192,11 @@ static void test_ldb_add_key_len_gt_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
- assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+ assert_int_equal(ret, LDB_SUCCESS);
talloc_free(tmp_ctx);
}
@@ -204,7 +218,8 @@ static void test_ldb_add_key_len_eq_max(void **state)
assert_non_null(msg);
/*
- * The zero terminator is part of the key
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
*/
xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
@@ -217,12 +232,145 @@ static void test_ldb_add_key_len_eq_max(void **state)
ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
assert_int_equal(ret, 0);
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
ret = ldb_add(test_ctx->ldb, msg);
assert_int_equal(ret, 0);
talloc_free(tmp_ctx);
}
+static int ldbtest_setup_noguid(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ *state = test_ctx;
+ return 0;
+}
+
+static void test_ldb_add_special_key_len_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_PROTOCOL_ERROR);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_special_key_len_eq_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 5; /* "dn=@" and the zero terminator */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "@%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
+static void test_ldb_add_dn_no_guid_mode(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = LMDB_MAX_KEY_SIZE - 7; /* "dn=dc=" and the zero terminator */
+ xs_size += 1; /* want key on char too long */
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_ERR_UNWILLING_TO_PERFORM);
+
+ talloc_free(tmp_ctx);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -234,6 +382,18 @@ int main(int argc, const char **argv)
test_ldb_add_key_len_gt_max,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_eq_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_special_key_len_gt_max,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_ldb_add_dn_no_guid_mode,
+ ldbtest_setup_noguid,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.11.0
From a71ce4d3f05fb78f8249e94328fcb76629b4c261 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 7 Mar 2018 12:05:34 +1300
Subject: [PATCH 09/24] ldb_mdb: Wrap mdb_env_open
Wrap mdb_env_open to ensure that we only have one MDB_env opened per
database in each process
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 144 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 112 insertions(+), 32 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 139140bb621..463ac43bea2 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -649,74 +649,155 @@ static int lmdb_pvt_destructor(struct lmdb_private *lmdb)
trans_finished(lmdb, ltx);
ltx = lmdb_private_trans_head(lmdb);
}
-
- mdb_env_close(lmdb->env);
lmdb->env = NULL;
return 0;
}
-static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
+struct mdb_env_wrap {
+ struct mdb_env_wrap *next, *prev;
+ dev_t device;
+ ino_t inode;
+ MDB_env *env;
+ int pid;
+};
+
+static struct mdb_env_wrap *mdb_list;
+
+/* destroy the last connection to an mdb */
+static int mdb_env_wrap_destructor(struct mdb_env_wrap *w)
+{
+ mdb_env_close(w->env);
+ DLIST_REMOVE(mdb_list, w);
+ return 0;
+}
+
+static int lmdb_open_env(TALLOC_CTX *mem_ctx,
+ MDB_env **env,
struct ldb_context *ldb,
const char *path,
- unsigned int flags,
- struct lmdb_private *lmdb)
+ unsigned int flags)
{
int ret;
- unsigned int mdb_flags;
- int lmdb_max_key_length;
-
- if (flags & LDB_FLG_DONT_CREATE_DB) {
- struct stat st;
+ unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
+ /*
+ * MDB_NOSUBDIR implies there is a separate file called path and a
+ * separate lockfile called path-lock
+ */
- if (stat(path, &st) != 0) {
- return LDB_ERR_UNAVAILABLE;
+ struct mdb_env_wrap *w;
+ struct stat st;
+
+ if (stat(path, &st) == 0) {
+ for (w=mdb_list;w;w=w->next) {
+ if (st.st_dev == w->device && st.st_ino == w->inode) {
+ /*
+ * We must have only one MDB_env per process
+ */
+ if (!talloc_reference(mem_ctx, w)) {
+ return ldb_oom(ldb);
+ }
+ *env = w->env;
+ return LDB_SUCCESS;
+ }
}
}
- ret = mdb_env_create(&lmdb->env);
+ w = talloc(mem_ctx, struct mdb_env_wrap);
+ if (w == NULL) {
+ return ldb_oom(ldb);
+ }
+
+ ret = mdb_env_create(env);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
- /* Close when lmdb is released */
- talloc_set_destructor(lmdb, lmdb_pvt_destructor);
-
- ret = mdb_env_set_mapsize(lmdb->env, 16LL * GIGABYTE);
+ /*
+ * Currently we set a 16Gb maximum database size
+ */
+ ret = mdb_env_set_mapsize(*env, 16LL * GIGABYTE);
if (ret != 0) {
ldb_asprintf_errstring(
ldb,
"Could not open MDB environment %s: %s\n",
path,
mdb_strerror(ret));
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
- mdb_env_set_maxreaders(lmdb->env, 100000);
- /* MDB_NOSUBDIR implies there is a separate file called path and a
- * separate lockfile called path-lock
+ mdb_env_set_maxreaders(*env, 100000);
+ /*
+ * As we ensure that there is only one MDB_env open per database per
+ * process. We can not use the MDB_RDONLY flag, as another ldb may be
+ * opened in read write mode
*/
- mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
- if (flags & LDB_FLG_RDONLY) {
- mdb_flags |= MDB_RDONLY;
- }
if (flags & LDB_FLG_NOSYNC) {
mdb_flags |= MDB_NOSYNC;
}
- ret = mdb_env_open(lmdb->env, path, mdb_flags, 0644);
+ ret = mdb_env_open(*env, path, mdb_flags, 0644);
if (ret != 0) {
ldb_asprintf_errstring(ldb,
"Could not open DB %s: %s\n",
path, mdb_strerror(ret));
- talloc_free(lmdb);
+ TALLOC_FREE(w);
return ldb_mdb_err_map(ret);
}
+ if (stat(path, &st) != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not stat %s:\n",
+ path);
+ TALLOC_FREE(w);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+ w->env = *env;
+ w->device = st.st_dev;
+ w->inode = st.st_ino;
+
+ talloc_set_destructor(w, mdb_env_wrap_destructor);
+
+ DLIST_ADD(mdb_list, w);
+
+ return LDB_SUCCESS;
+
+}
+
+static int lmdb_pvt_open(struct lmdb_private *lmdb,
+ struct ldb_context *ldb,
+ const char *path,
+ unsigned int flags)
+{
+ int ret;
+ int lmdb_max_key_length;
+
+ if (flags & LDB_FLG_DONT_CREATE_DB) {
+ struct stat st;
+ if (stat(path, &st) != 0) {
+ return LDB_ERR_UNAVAILABLE;
+ }
+ }
+
+ ret = lmdb_open_env(lmdb, &lmdb->env, ldb, path, flags);
+ if (ret != 0) {
+ ldb_asprintf_errstring(
+ ldb,
+ "Could not create MDB environment %s: %s\n",
+ path,
+ mdb_strerror(ret));
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
+
+ /* Close when lmdb is released */
+ talloc_set_destructor(lmdb, lmdb_pvt_destructor);
+
/* Store the original pid during the LMDB open */
lmdb->pid = getpid();
@@ -724,7 +805,6 @@ static int lmdb_pvt_open(TALLOC_CTX *mem_ctx,
/* This will never happen, but if it does make sure to freak out */
if (lmdb_max_key_length < LDB_MDB_MAX_KEY_LENGTH) {
- talloc_free(lmdb);
return ldb_operr(ldb);
}
@@ -760,17 +840,17 @@ int lmdb_connect(struct ldb_context *ldb,
return LDB_ERR_OPERATIONS_ERROR;
}
- lmdb = talloc_zero(ldb, struct lmdb_private);
+ lmdb = talloc_zero(ltdb, struct lmdb_private);
if (lmdb == NULL) {
TALLOC_FREE(ltdb);
- ldb_oom(ldb);
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_oom(ldb);
}
lmdb->ldb = ldb;
ltdb->kv_ops = &lmdb_key_value_ops;
- ret = lmdb_pvt_open(ldb, ldb, path, flags, lmdb);
+ ret = lmdb_pvt_open(lmdb, ldb, path, flags);
if (ret != LDB_SUCCESS) {
+ TALLOC_FREE(ltdb);
return ret;
}
--
2.11.0
From 8a860f3e519ded0c23fea23fa4a1cafeea2f9ed1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 4 May 2018 14:35:14 +1200
Subject: [PATCH 10/24] ldb_tdb: Prevent ldb_tdb reuse after a fork()
We may relax this restriction in the future, but for now do not assume
that the caller has done a tdb_reopen_all() at the right time.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 115 +++++++++++++++++++++++++++++++++++++++++-----
lib/ldb/ldb_tdb/ldb_tdb.h | 7 +++
2 files changed, 111 insertions(+), 11 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 0833a4fd0ca..24a5e7b6c16 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -100,7 +100,18 @@ static int ltdb_lock_read(struct ldb_module *module)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int tdb_ret = 0;
int ret;
-
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
if (tdb_transaction_active(ltdb->tdb) == false &&
ltdb->read_lock_count == 0) {
tdb_ret = tdb_lockall_read(ltdb->tdb);
@@ -128,6 +139,17 @@ static int ltdb_unlock_read(struct ldb_module *module)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
tdb_unlockall_read(ltdb->tdb);
ltdb->read_lock_count--;
@@ -1447,21 +1469,69 @@ static int ltdb_rename(struct ltdb_context *ctx)
static int ltdb_tdb_transaction_start(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_start(ltdb->tdb);
}
static int ltdb_tdb_transaction_cancel(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_cancel(ltdb->tdb);
}
static int ltdb_tdb_transaction_prepare_commit(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_prepare_commit(ltdb->tdb);
}
static int ltdb_tdb_transaction_commit(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_commit(ltdb->tdb);
}
@@ -1470,6 +1540,18 @@ static int ltdb_start_trans(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
/* Do not take out the transaction lock on a read-only DB */
if (ltdb->read_only) {
return LDB_ERR_UNWILLING_TO_PERFORM;
@@ -1498,7 +1580,18 @@ static int ltdb_prepare_commit(struct ldb_module *module)
int ret;
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
-
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
if (!ltdb->kv_ops->transaction_active(ltdb)) {
ldb_set_errstring(ldb_module_get_ctx(module),
"ltdb_prepare_commit() called "
@@ -2138,8 +2231,6 @@ int init_store(struct ltdb_private *ltdb,
const char *options[],
struct ldb_module **_module)
{
- struct ldb_module *module;
-
if (getenv("LDB_WARN_UNINDEXED")) {
ltdb->warn_unindexed = true;
}
@@ -2150,23 +2241,25 @@ int init_store(struct ltdb_private *ltdb,
ltdb->sequence_number = 0;
- module = ldb_module_new(ldb, ldb, name, <db_ops);
- if (!module) {
+ ltdb->pid = getpid();
+
+ ltdb->module = ldb_module_new(ldb, ldb, name, <db_ops);
+ if (!ltdb->module) {
ldb_oom(ldb);
talloc_free(ltdb);
return LDB_ERR_OPERATIONS_ERROR;
}
- ldb_module_set_private(module, ltdb);
- talloc_steal(module, ltdb);
+ ldb_module_set_private(ltdb->module, ltdb);
+ talloc_steal(ltdb->module, ltdb);
- if (ltdb_cache_load(module) != 0) {
+ if (ltdb_cache_load(ltdb->module) != 0) {
ldb_asprintf_errstring(ldb, "Unable to load ltdb cache "
"records for backend '%s'", name);
- talloc_free(module);
+ talloc_free(ltdb->module);
return LDB_ERR_OPERATIONS_ERROR;
}
- *_module = module;
+ *_module = ltdb->module;
/*
* Set or override the maximum key length
*
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 9c3f8d89d8d..f6819d7e325 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
ldb_context */
struct ltdb_private {
const struct kv_db_ops *kv_ops;
+ struct ldb_module *module;
TDB_CONTEXT *tdb;
struct lmdb_private *lmdb_private;
unsigned int connect_flags;
@@ -76,6 +77,12 @@ struct ltdb_private {
* greater than this length will be rejected.
*/
unsigned max_key_length;
+
+ /*
+ * The PID that opened this database so we don't work in a
+ * fork()ed child.
+ */
+ pid_t pid;
};
struct ltdb_context {
--
2.11.0
From fa2eff52a982144c0ff1aeccdc9422758f1867d8 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Wed, 21 Mar 2018 11:38:22 +1300
Subject: [PATCH 11/24] ldb_mdb: handle EBADE from mdb_env_open
Under some circumstances mdb_env_open returns EBADE, we treat this as
indicating the file is not a valid lmdb format file.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 463ac43bea2..4e9fbfa4d72 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -40,6 +40,7 @@ int ldb_mdb_err_map(int lmdb_err)
return LDB_SUCCESS;
case EIO:
return LDB_ERR_OPERATIONS_ERROR;
+ case EBADE:
case MDB_INCOMPATIBLE:
case MDB_CORRUPTED:
case MDB_INVALID:
@@ -792,7 +793,7 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
"Could not create MDB environment %s: %s\n",
path,
mdb_strerror(ret));
- return LDB_ERR_OPERATIONS_ERROR;
+ return ldb_mdb_err_map(ret);
}
/* Close when lmdb is released */
--
2.11.0
From e38f1bab41ff9b2be1b73736dda85074919a4deb Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 12/24] ldb_mdb: prevent MDB_env reuse across forks
MDB_env's may not be reused accross forks. Check the pid that the lmdb
structure was created by, and return an error if it is being used by a
different process.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 33 +++++++++++++++++++++++++++++++--
1 file changed, 31 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 4e9fbfa4d72..9bdeb467962 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -443,6 +443,18 @@ static int lmdb_lock_read(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct lmdb_private *lmdb = ltdb->lmdb_private;
+ pid_t pid = getpid();
+
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
lmdb->error = MDB_SUCCESS;
if (lmdb_transaction_active(ltdb) == false &&
@@ -482,6 +494,7 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
struct lmdb_trans *ltx;
struct lmdb_trans *ltx_head;
MDB_txn *tx_parent;
+ pid_t pid = getpid();
/* Do not take out the transaction lock on a read-only DB */
if (ltdb->read_only) {
@@ -493,6 +506,18 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
return ldb_oom(lmdb->ldb);
}
+ if (pid != lmdb->pid) {
+ ldb_asprintf_errstring(
+ lmdb->ldb,
+ __location__": Reusing ldb opened by pid %d in "
+ "process %d\n",
+ lmdb->pid,
+ pid);
+ lmdb->error = MDB_BAD_TXN;
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
+
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
@@ -660,7 +685,7 @@ struct mdb_env_wrap {
dev_t device;
ino_t inode;
MDB_env *env;
- int pid;
+ pid_t pid;
};
static struct mdb_env_wrap *mdb_list;
@@ -688,10 +713,13 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
struct mdb_env_wrap *w;
struct stat st;
+ pid_t pid = getpid();
if (stat(path, &st) == 0) {
for (w=mdb_list;w;w=w->next) {
- if (st.st_dev == w->device && st.st_ino == w->inode) {
+ if (st.st_dev == w->device &&
+ st.st_ino == w->inode &&
+ pid == w->pid) {
/*
* We must have only one MDB_env per process
*/
@@ -762,6 +790,7 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
w->env = *env;
w->device = st.st_dev;
w->inode = st.st_ino;
+ w->pid = pid;
talloc_set_destructor(w, mdb_env_wrap_destructor);
--
2.11.0
From 1972c02b122c4c350c55ebd0ce514fd58cd8da6c Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 13/24] ldb/tests: add tests for
transaction_{start,commit}/lock_read across forks
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 216 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 216 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 1340f5efa23..78084ef3c9d 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3820,6 +3820,210 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
}
}
+static void test_transaction_start_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ close(pipes[0]);
+ ret = ldb_transaction_start(ldb1);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_transaction_start "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_transaction_commit_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_transaction_start(ldb1);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ close(pipes[0]);
+ ret = ldb_transaction_commit(ldb1);
+
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_transaction_commit "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_lock_read_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_search "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+
+ {
+ /*
+ * Ensure that the search actually succeeds on the opening
+ * pid
+ */
+ struct ldb_dn *basedn;
+ struct ldb_result *result = NULL;
+
+ close(pipes[0]);
+
+ basedn = ldb_dn_new_fmt(test_ctx, test_ctx->ldb, "dc=test");
+ assert_non_null(basedn);
+
+ ret = ldb_search(test_ctx->ldb,
+ test_ctx,
+ &result,
+ basedn,
+ LDB_SCOPE_BASE,
+ NULL,
+ NULL);
+ assert_int_equal(0, ret);
+ }
+}
int main(int argc, const char **argv)
{
@@ -3984,6 +4188,18 @@ int main(int argc, const char **argv)
test_ldb_talloc_destructor_transaction_cleanup,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_start_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_transaction_commit_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_lock_read_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.11.0
From d6dd6354c94205bec9a68e93ceb6fb30968d236e Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Thu, 8 Mar 2018 16:47:59 +1300
Subject: [PATCH 14/24] ldb_mdb/tests: Tests for wrap open
Tests to ensure that the mdb_env wrapping code correctly handles
multiple ldb's point to the same physical database file.
The test_ldb_close_with_multiple_connections tests are in
ldb_mod_op_test due to the utility code it uses from
elsewhere in that test.
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 69 +++++++++++++++-
lib/ldb/tests/ldb_mod_op_test.c | 173 ++++++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 5 +-
3 files changed, 243 insertions(+), 4 deletions(-)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index eece2be8057..f2b04ecd0d7 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -55,8 +55,8 @@
#include <sys/wait.h>
-#include <lmdb.h>
-
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
#define TEST_BE "mdb"
@@ -371,6 +371,67 @@ static void test_ldb_add_dn_no_guid_mode(void **state)
talloc_free(tmp_ctx);
}
+static struct MDB_env *get_mdb_env(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct MDB_env *env = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ lmdb = ltdb->lmdb_private;
+ assert_non_null(lmdb);
+
+ env = lmdb->env;
+ assert_non_null(env);
+
+ return env;
+}
+
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ struct MDB_env *env3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+ env3 = get_mdb_env(ldb3);
+
+ assert_ptr_equal(env1, env2);
+ assert_ptr_equal(env1, env3);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -394,6 +455,10 @@ int main(int argc, const char **argv)
test_ldb_add_dn_no_guid_mode,
ldbtest_setup_noguid,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 78084ef3c9d..0141640b1e5 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -38,6 +38,12 @@
#define TEST_BE DEFAULT_BE
#endif /* TEST_BE */
+#ifdef TEST_LMDB
+#include "lmdb.h"
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
+#endif
+
struct ldbtest_ctx {
struct tevent_context *ev;
struct ldb_context *ldb;
@@ -3820,6 +3826,167 @@ static void test_ldb_talloc_destructor_transaction_cleanup(void **state)
}
}
+#ifdef TEST_LMDB
+static int test_ldb_multiple_connections_callback(struct ldb_request *req,
+ struct ldb_reply *ares)
+{
+ int ret;
+ int pipes[2];
+ char buf[2];
+ int pid, child_pid;
+ int wstatus;
+
+ switch (ares->type) {
+ case LDB_REPLY_ENTRY:
+ break;
+
+ case LDB_REPLY_REFERRAL:
+ return LDB_SUCCESS;
+
+ case LDB_REPLY_DONE:
+ return ldb_request_done(req, LDB_SUCCESS);
+ }
+
+ {
+ /*
+ * We open a new ldb on an ldb that is already open and
+ * then close it.
+ *
+ * If the multiple connection wrapping is correct the
+ * underlying MDB_env will be left open and we should see
+ * an active reader in the child we fork next
+ */
+ struct ldb_context *ldb = NULL;
+ struct tevent_context *ev = NULL;
+ TALLOC_CTX *mem_ctx = talloc_new(NULL);
+
+ ev = tevent_context_init(mem_ctx);
+ assert_non_null(ev);
+
+ ldb = ldb_init(mem_ctx, ev);
+ assert_non_null(ldb);
+
+ ret = ldb_connect(ldb, TEST_BE"://apitest.ldb" , 0, NULL);
+ if (ret != LDB_SUCCESS) {
+ return ret;
+ }
+ TALLOC_FREE(ldb);
+ TALLOC_FREE(mem_ctx);
+ }
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct MDB_env *env = NULL;
+ struct MDB_envinfo stat;
+ close(pipes[0]);
+
+ /*
+ * Check that there are exactly two readers on the MDB file
+ * backing the ldb.
+ *
+ */
+ ret = mdb_env_create(&env);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_create returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_open(env,
+ "apitest.ldb",
+ MDB_NOSUBDIR | MDB_NOTLS,
+ 0644);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_open returned (%d)",
+ ret);
+ exit(ret);
+ }
+
+ ret = mdb_env_info(env, &stat);
+ if (ret != 0) {
+ print_error(__location__
+ " mdb_env_info returned (%d)",
+ ret);
+ exit(ret);
+ }
+ if (stat.me_numreaders != 2) {
+ print_error(__location__
+ " Incorrect number of readers (%d)",
+ stat.me_numreaders);
+ exit(LDB_ERR_CONSTRAINT_VIOLATION);
+ }
+
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+ return LDB_SUCCESS;
+
+}
+
+static void test_ldb_close_with_multiple_connections(void **state)
+{
+ struct search_test_ctx *search_test_ctx = NULL;
+ struct ldb_dn *search_dn = NULL;
+ struct ldb_request *req = NULL;
+ int ret = 0;
+
+ search_test_ctx = talloc_get_type_abort(*state, struct search_test_ctx);
+ assert_non_null(search_test_ctx);
+
+ search_dn = ldb_dn_new_fmt(search_test_ctx,
+ search_test_ctx->ldb_test_ctx->ldb,
+ "cn=test_search_cn,"
+ "dc=search_test_entry");
+ assert_non_null(search_dn);
+
+ /*
+ * The search just needs to call DONE, we don't care about the
+ * contents of the search for this test
+ */
+ ret = ldb_build_search_req(&req,
+ search_test_ctx->ldb_test_ctx->ldb,
+ search_test_ctx,
+ search_dn,
+ LDB_SCOPE_SUBTREE,
+ "(&(!(filterAttr=*))"
+ "(cn=test_search_cn))",
+ NULL,
+ NULL,
+ NULL,
+ test_ldb_multiple_connections_callback,
+ NULL);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_request(search_test_ctx->ldb_test_ctx->ldb, req);
+ assert_int_equal(ret, 0);
+
+ ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+ assert_int_equal(ret, 0);
+}
+
+#endif
+
static void test_transaction_start_across_fork(void **state)
{
struct ldb_context *ldb1 = NULL;
@@ -4188,6 +4355,12 @@ int main(int argc, const char **argv)
test_ldb_talloc_destructor_transaction_cleanup,
ldbtest_setup,
ldbtest_teardown),
+#ifdef TEST_LMDB
+ cmocka_unit_test_setup_teardown(
+ test_ldb_close_with_multiple_connections,
+ ldb_search_test_setup,
+ ldb_search_test_teardown),
+#endif
cmocka_unit_test_setup_teardown(
test_transaction_start_across_fork,
ldbtest_setup,
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 621cfcf4b77..23eb02d0479 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -443,8 +443,9 @@ def build(bld):
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
- cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1',
- deps='cmocka ldb',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 '
+ + '-DTEST_LMDB=1',
+ deps='cmocka ldb lmdb',
install=False)
bld.SAMBA_BINARY('ldb_lmdb_test',
--
2.11.0
From f0e95dd9db66825ddad3b16251c1aaba4cc6ed9d Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Fri, 23 Mar 2018 11:29:25 +1300
Subject: [PATCH 15/24] ldb_mdb: Remove implicit read lock and remove
transaction counter
The way to know if we are in a transaction is if there is a non-NULL
transaction handle.
This allows the ldb_mdb_kv_ops_test test to be run.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
---
lib/ldb/ldb_mdb/ldb_mdb.c | 43 ++++++++-----------------------------------
lib/ldb/wscript | 1 +
2 files changed, 9 insertions(+), 35 deletions(-)
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 9bdeb467962..bbd0057bf56 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -129,29 +129,21 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
return ltx;
}
+
static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
{
MDB_txn *txn = NULL;
+ txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
+ if (txn != NULL) {
+ return txn;
+ }
if (lmdb->read_txn != NULL) {
return lmdb->read_txn;
}
-
- txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
- if (txn == NULL) {
- int ret;
- ret = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &txn);
- if (ret != 0) {
- lmdb->error = ret;
- ldb_asprintf_errstring(lmdb->ldb,
- "%s failed: %s\n", __FUNCTION__,
- mdb_strerror(ret));
- return NULL;
- }
- lmdb->read_txn = txn;
- }
-
- return txn;
+ lmdb->error = MDB_BAD_TXN;
+ ldb_set_errstring(lmdb->ldb, __location__":No active transaction\n");
+ return NULL;
}
static int lmdb_store(struct ltdb_private *ltdb,
@@ -197,10 +189,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
MDB_val value;
lmdb->error = mdb_get(txn, dbi, &mdb_key, &value);
if (lmdb->error != MDB_SUCCESS) {
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
mdb_flags = 0;
@@ -304,11 +292,6 @@ done:
mdb_cursor_close(cursor);
}
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
-
if (lmdb->error != MDB_SUCCESS) {
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
@@ -414,10 +397,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
if (lmdb->error != MDB_SUCCESS) {
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
if (lmdb->error == MDB_NOTFOUND) {
return LDB_ERR_NO_SUCH_OBJECT;
}
@@ -429,11 +408,6 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
- /* We created a read transaction, commit it */
- if (ltdb->read_lock_count == 0 && lmdb->read_txn != NULL) {
- mdb_txn_commit(lmdb->read_txn);
- lmdb->read_txn = NULL;
- }
return parser(key, data, ctx);
}
@@ -517,7 +491,6 @@ static int lmdb_transaction_start(struct ltdb_private *ltdb)
return LDB_ERR_PROTOCOL_ERROR;
}
-
ltx_head = lmdb_private_trans_head(lmdb);
tx_parent = lmdb_trans_get_tx(ltx_head);
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 23eb02d0479..36f1bb4d58b 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -513,6 +513,7 @@ def test(ctx):
# we don't want to run ldb_lmdb_size_test (which proves we can
# fit > 4G of data into the DB), it would fill up the disk on
# many of our test instances
+ test_exes.append('ldb_mdb_kv_ops_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
cmocka_ret = cmocka_ret or samba_utils.RUN_COMMAND(cmd)
--
2.11.0
From bdcc7bebd5fb7cb71fb3f3206e9f62aed9b71767 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 08:14:09 +1300
Subject: [PATCH 16/24] ldb_mdb/tests: test large index key value
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 44 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index f2b04ecd0d7..e8f8e4f8ab1 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -201,6 +201,46 @@ static void test_ldb_add_key_len_gt_max(void **state)
talloc_free(tmp_ctx);
}
+static void test_ldb_add_key_len_2x_gt_max(void **state)
+{
+ int ret;
+ int xs_size = 0;
+ struct ldb_message *msg;
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ char *xs = NULL;
+ TALLOC_CTX *tmp_ctx;
+
+ tmp_ctx = talloc_new(test_ctx);
+ assert_non_null(tmp_ctx);
+
+ msg = ldb_msg_new(tmp_ctx);
+ assert_non_null(msg);
+
+ /*
+ * The zero terminator is part of the key if we were not in
+ * GUID mode
+ */
+
+ xs_size = 2 * LMDB_MAX_KEY_SIZE;
+ xs = talloc_zero_size(tmp_ctx, (xs_size + 1));
+ memset(xs, 'x', xs_size);
+
+ msg->dn = ldb_dn_new_fmt(msg, test_ctx->ldb, "dc=%s", xs);
+ assert_non_null(msg->dn);
+
+ ret = ldb_msg_add_string(msg, "cn", "test_cn_val");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_msg_add_string(msg, "objectUUID", "0123456789abcdef");
+ assert_int_equal(ret, 0);
+
+ ret = ldb_add(test_ctx->ldb, msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+
+ talloc_free(tmp_ctx);
+}
+
static void test_ldb_add_key_len_eq_max(void **state)
{
int ret;
@@ -444,6 +484,10 @@ int main(int argc, const char **argv)
ldbtest_setup,
ldbtest_teardown),
cmocka_unit_test_setup_teardown(
+ test_ldb_add_key_len_2x_gt_max,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
test_ldb_add_special_key_len_eq_max,
ldbtest_setup_noguid,
ldbtest_teardown),
--
2.11.0
From 5b85effbc1b1978c8ff09d7432534506da2f8db6 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 13 Mar 2018 15:08:10 +1300
Subject: [PATCH 17/24] ldb_mdb/tests: add tests for multiple opens across
forks
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_lmdb_test.c | 79 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index e8f8e4f8ab1..a254a849f4a 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -472,6 +472,81 @@ static void test_multiple_opens(void **state)
assert_ptr_equal(env1, env3);
}
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ env3 = get_mdb_env(ldb3);
+ if (env1 != env2) {
+ print_error(__location__": env1 != env2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (env1 == env3) {
+ print_error(__location__": env1 == env3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -503,6 +578,10 @@ int main(int argc, const char **argv)
test_multiple_opens,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.11.0
From 5f8df36adba1b65f91bda9f754936bbe260c33e5 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 7 May 2018 12:59:49 +1200
Subject: [PATCH 18/24] ldb: Reset errno before checking it in ltdb_connect()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 24a5e7b6c16..4747deb8ca7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -2355,6 +2355,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
ltdb->kv_ops = &key_value_ops;
+ errno = 0;
/* note that we use quite a large default hash size */
ltdb->tdb = ltdb_wrap_open(ltdb, path, 10000,
tdb_flags, open_flags,
--
2.11.0
From 397d1eaaecec6f6a2d04b2cb34c2353fd42365ad Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 4 May 2018 22:22:26 +1200
Subject: [PATCH 19/24] ldb_tdb: Allow use of a TDB for ldb_tdb after as fork()
Otherwise we rely on the caller doing tdb_reopen_all() which should
not be their job.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb_wrap.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb_wrap.c b/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
index eb168098a75..4b94f820b59 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
@@ -74,6 +74,7 @@ struct ltdb_wrap {
struct tdb_context *tdb;
dev_t device;
ino_t inode;
+ pid_t pid;
};
static struct ltdb_wrap *tdb_list;
@@ -105,9 +106,25 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
if (stat(path, &st) == 0) {
for (w=tdb_list;w;w=w->next) {
if (st.st_dev == w->device && st.st_ino == w->inode) {
+ pid_t pid = getpid();
+ int ret;
if (!talloc_reference(mem_ctx, w)) {
return NULL;
}
+ if (w->pid != pid) {
+ ret = tdb_reopen(w->tdb);
+ if (ret != 0) {
+ /*
+ * Avoid use-after-free:
+ * on fail the TDB
+ * is closed!
+ */
+ DLIST_REMOVE(tdb_list,
+ w);
+ return NULL;
+ }
+ w->pid = pid;
+ }
return w->tdb;
}
}
@@ -135,6 +152,7 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
w->device = st.st_dev;
w->inode = st.st_ino;
+ w->pid = getpid();
talloc_set_destructor(w, ltdb_wrap_destructor);
--
2.11.0
From 16c2ace47fca613b591ac197f8a24e2cd1142cd1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Mon, 7 May 2018 12:59:00 +1200
Subject: [PATCH 20/24] ldb: Add tests for ldb_tdb use after a fork()
We need to show that despite the internal cache of TDB pointers that it
is safe to open a ldb_tdb after a fork()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_tdb_test.c | 387 +++++++++++++++++++++++++++++++++++++++++++
lib/ldb/wscript | 5 +
2 files changed, 392 insertions(+)
create mode 100644 lib/ldb/tests/ldb_tdb_test.c
diff --git a/lib/ldb/tests/ldb_tdb_test.c b/lib/ldb/tests/ldb_tdb_test.c
new file mode 100644
index 00000000000..abba8cb8f69
--- /dev/null
+++ b/lib/ldb/tests/ldb_tdb_test.c
@@ -0,0 +1,387 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+#include <tevent.h>
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "../ldb_tdb/ldb_tdb.h"
+
+#define TEST_BE "tdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+
+static TDB_CONTEXT *get_tdb_context(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ TDB_CONTEXT *tdb = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ tdb = ltdb->tdb;
+ assert_non_null(tdb);
+
+ return tdb;
+}
+
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ TDB_CONTEXT *tdb3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+ tdb3 = get_tdb_context(ldb3);
+
+ assert_ptr_equal(tdb1, tdb2);
+ assert_ptr_equal(tdb1, tdb3);
+}
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ TDB_CONTEXT *tdb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ tdb3 = get_tdb_context(ldb3);
+ if (tdb1 != tdb2) {
+ print_error(__location__": tdb1 != tdb2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (tdb1 != tdb3) {
+ print_error(__location__": tdb1 != tdb3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_multiple_opens_across_fork_triggers_reopen(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+ assert_ptr_equal(tdb1, tdb2);
+
+ /*
+ * Break the internal tdb_reopen() by making a
+ * transaction
+ *
+ * This shows that the tdb_reopen() is called, which is
+ * essential if the host OS does not have pread()
+ */
+ ret = tdb_transaction_start(tdb1);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+
+ /*
+ * This should fail as we have taken out a lock
+ * against the raw TDB above, and tdb_reopen()
+ * will fail in that state.
+ *
+ * This check matters as tdb_reopen() is important
+ * if the host does not have pread()
+ */
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret == 0) {
+ print_error(__location__": ldb_connect expected "
+ "LDB_ERR_OPERATIONS_ERROR "
+ "returned (%d)\n",
+ ret);
+ exit(5000);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork_triggers_reopen,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 36f1bb4d58b..7557ecb075b 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -440,6 +440,11 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_tdb_test',
+ source='tests/ldb_tdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
--
2.11.0
From 865479617e05dc20f81d16e3c5c4d0a278c70ff7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 4 May 2018 22:22:53 +1200
Subject: [PATCH 21/24] ldb: Ensure we can open a new LDB after a fork()
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/tests/ldb_mod_op_test.c | 64 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 64 insertions(+)
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index 0141640b1e5..7b6f19c3a18 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -4192,6 +4192,66 @@ static void test_lock_read_across_fork(void **state)
}
}
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -4373,6 +4433,10 @@ int main(int argc, const char **argv)
test_lock_read_across_fork,
ldbtest_setup,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
--
2.11.0
From 4c9396a9f281993145bf9a78da55a9efa4ee2d43 Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 15:11:23 +1300
Subject: [PATCH 22/24] ldb: Add MDB support to ldb://
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Stefan Metzmacher <metze at samba.org>
---
lib/ldb/ldb_ldb/ldb_ldb.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
index 3e4398f5c17..a5a36121a9f 100644
--- a/lib/ldb/ldb_ldb/ldb_ldb.c
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -19,6 +19,9 @@
*/
#include "ldb_private.h"
#include "../ldb_tdb/ldb_tdb.h"
+#ifdef HAVE_LMDB
+#include "../ldb_mdb/ldb_mdb.h"
+#endif /* HAVE_LMDB */
/*
connect to the database
@@ -50,6 +53,22 @@ static int lldb_connect(struct ldb_context *ldb,
* Don't create the database if it's not there
*/
flags |= LDB_FLG_DONT_CREATE_DB;
+#ifdef HAVE_LMDB
+ /*
+ * Try opening the database as an lmdb
+ */
+ ret = lmdb_connect(ldb, path, flags, options, module);
+ if (ret == LDB_SUCCESS) {
+ return ret;
+ }
+ if (ret != LDB_ERR_UNAVAILABLE) {
+ return ret;
+ }
+
+ /*
+ * Not mdb so try as tdb
+ */
+#endif /* HAVE_LMDB */
ret = ltdb_connect(ldb, path, flags, options, module);
return ret;
}
--
2.11.0
From b06796c09b9a7e9ef5036b56d57ae39a8aad823d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Tue, 20 Mar 2018 12:58:02 +1300
Subject: [PATCH 23/24] ldb: Release ldb 1.4.0
* New LMDB backend
* Comprehensive tests for index behaviour
* Enforce transactions for writes
* Enforce read lock use for all reads
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ABI/ldb-1.4.0.sigs | 279 ++++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.4.0.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 284 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.4.0.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
diff --git a/lib/ldb/ABI/ldb-1.4.0.sigs b/lib/ldb/ABI/ldb-1.4.0.sigs
new file mode 100644
index 00000000000..a31b84ef4b5
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.4.0.sigs
@@ -0,0 +1,279 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handle_use_global_event_context: void (struct ldb_handle *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t)
+ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_read_lock: int (struct ldb_module *)
+ldb_next_read_unlock: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_require_private_event_context: void (struct ldb_context *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.4.0.sigs b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.4.0.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 7557ecb075b..46472db0e6b 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.3.2'
+VERSION = '1.4.0'
blddir = 'bin'
--
2.11.0
From 4b097de0606d53a8d6d29b0bed2269f0bf9c992f Mon Sep 17 00:00:00 2001
From: Gary Lockyer <gary at catalyst.net.nz>
Date: Tue, 6 Mar 2018 13:40:21 +1300
Subject: [PATCH 24/24] ldb-samba: Handle generic mdb:// url scheme in
ldb_relative_path()
Signed-off-by: Gary Lockyer <gary at catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet at samba.org>
Reviewed-by: Stefan Metzmacher <metze at samba.org>
---
lib/ldb-samba/ldb_wrap.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index da383d279c4..34148a13ab3 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -356,6 +356,8 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
}
if (strncmp("tdb://", base_url, 6) == 0) {
base_url = base_url+6;
+ } else if (strncmp("mdb://", base_url, 6) == 0) {
+ base_url = base_url+6;
} else if (strncmp("ldb://", base_url, 6) == 0) {
base_url = base_url+6;
}
--
2.11.0
-------------- next part --------------
diff --git a/lib/ldb-samba/ldb_wrap.c b/lib/ldb-samba/ldb_wrap.c
index da383d279c4..34148a13ab3 100644
--- a/lib/ldb-samba/ldb_wrap.c
+++ b/lib/ldb-samba/ldb_wrap.c
@@ -356,6 +356,8 @@ int samba_ldb_connect(struct ldb_context *ldb, struct loadparm_context *lp_ctx,
}
if (strncmp("tdb://", base_url, 6) == 0) {
base_url = base_url+6;
+ } else if (strncmp("mdb://", base_url, 6) == 0) {
+ base_url = base_url+6;
} else if (strncmp("ldb://", base_url, 6) == 0) {
base_url = base_url+6;
}
diff --git a/lib/ldb/ldb_ldb/ldb_ldb.c b/lib/ldb/ldb_ldb/ldb_ldb.c
index a63967ff8d6..a5a36121a9f 100644
--- a/lib/ldb/ldb_ldb/ldb_ldb.c
+++ b/lib/ldb/ldb_ldb/ldb_ldb.c
@@ -19,7 +19,9 @@
*/
#include "ldb_private.h"
#include "../ldb_tdb/ldb_tdb.h"
+#ifdef HAVE_LMDB
#include "../ldb_mdb/ldb_mdb.h"
+#endif /* HAVE_LMDB */
/*
connect to the database
@@ -59,15 +61,15 @@ static int lldb_connect(struct ldb_context *ldb,
if (ret == LDB_SUCCESS) {
return ret;
}
- if (ret == LDB_ERR_UNAVAILABLE) {
- /*
- * Not mdb so try as tdb
- */
- ret = ltdb_connect(ldb, path, flags, options, module);
+ if (ret != LDB_ERR_UNAVAILABLE) {
+ return ret;
}
-#else
+
+ /*
+ * Not mdb so try as tdb
+ */
+#endif /* HAVE_LMDB */
ret = ltdb_connect(ldb, path, flags, options, module);
-#endif
return ret;
}
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index ce757f557cb..bbd0057bf56 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -22,7 +22,6 @@
License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
-#include "lmdb.h"
#include "ldb_mdb.h"
#include "../ldb_tdb/ldb_tdb.h"
#include "include/dlinklist.h"
@@ -32,7 +31,6 @@
#define LDB_MDB_MAX_KEY_LENGTH 511
-#define MEGABYTE (1024*1024)
#define GIGABYTE (1024*1024*1024)
int ldb_mdb_err_map(int lmdb_err)
@@ -93,7 +91,8 @@ static int lmdb_error_at(struct ldb_context *ldb,
}
-static bool lmdb_transaction_active(struct ltdb_private *ltdb) {
+static bool lmdb_transaction_active(struct ltdb_private *ltdb)
+{
return ltdb->lmdb_private->txlist != NULL;
}
@@ -133,7 +132,7 @@ static struct lmdb_trans *lmdb_private_trans_head(struct lmdb_private *lmdb)
static MDB_txn *get_current_txn(struct lmdb_private *lmdb)
{
- MDB_txn *txn;
+ MDB_txn *txn = NULL;
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn != NULL) {
@@ -161,6 +160,7 @@ static int lmdb_store(struct ltdb_private *ltdb,
if (ltdb->read_only) {
return LDB_ERR_UNWILLING_TO_PERFORM;
}
+
txn = lmdb_trans_get_tx(lmdb_private_trans_head(lmdb));
if (txn == NULL) {
ldb_debug(lmdb->ldb, LDB_DEBUG_FATAL, "No transaction");
@@ -196,7 +196,7 @@ static int lmdb_store(struct ltdb_private *ltdb,
mdb_flags = 0;
}
- lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
+ lmdb->error = mdb_put(txn, dbi, &mdb_key, &mdb_data, mdb_flags);
if (lmdb->error != MDB_SUCCESS) {
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
@@ -204,7 +204,6 @@ static int lmdb_store(struct ltdb_private *ltdb,
return ldb_mdb_err_map(lmdb->error);
}
-
static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
@@ -231,7 +230,7 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
mdb_key.mv_size = key.length;
mdb_key.mv_data = key.data;
- lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
+ lmdb->error = mdb_del(txn, dbi, &mdb_key, NULL);
if (lmdb->error != MDB_SUCCESS) {
return ldb_mdb_error(lmdb->ldb, lmdb->error);
}
@@ -239,7 +238,7 @@ static int lmdb_delete(struct ltdb_private *ltdb, struct ldb_val key)
}
static int lmdb_traverse_fn(struct ltdb_private *ltdb,
- ldb_kv_traverse_fn fn,
+ ldb_kv_traverse_fn fn,
void *ctx)
{
struct lmdb_private *lmdb = ltdb->lmdb_private;
@@ -335,6 +334,7 @@ static int lmdb_update_in_iterate(struct ltdb_private *ltdb,
ret = ldb_mdb_error(lmdb->ldb, lmdb->error);
goto done;
}
+
lmdb->error = lmdb_store(ltdb, key2, copy, 0);
if (lmdb->error != MDB_SUCCESS) {
ldb_debug(
@@ -364,6 +364,7 @@ done:
return ret;
}
+
/* Handles only a single record */
static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
int (*parser)(struct ldb_val key, struct ldb_val data,
@@ -392,7 +393,7 @@ static int lmdb_parse_record(struct ltdb_private *ltdb, struct ldb_val key,
mdb_key.mv_size = key.length;
mdb_key.mv_data = key.data;
- lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
+ lmdb->error = mdb_get(txn, dbi, &mdb_key, &mdb_data);
if (lmdb->error != MDB_SUCCESS) {
/* TODO closing a handle should not even be necessary */
mdb_dbi_close(lmdb->env, dbi);
@@ -674,7 +675,8 @@ static int lmdb_open_env(TALLOC_CTX *mem_ctx,
MDB_env **env,
struct ldb_context *ldb,
const char *path,
- unsigned int flags) {
+ unsigned int flags)
+{
int ret;
unsigned int mdb_flags = MDB_NOSUBDIR|MDB_NOTLS;
/*
@@ -810,7 +812,6 @@ static int lmdb_pvt_open(struct lmdb_private *lmdb,
}
return LDB_SUCCESS;
-
}
int lmdb_connect(struct ldb_context *ldb,
@@ -824,11 +825,11 @@ int lmdb_connect(struct ldb_context *ldb,
struct ltdb_private *ltdb = NULL;
int ret;
- /*
- * We hold locks, so we must use a private event context
- * on each returned handle
- */
- ldb_set_require_private_event_context(ldb);
+ /*
+ * We hold locks, so we must use a private event context
+ * on each returned handle
+ */
+ ldb_set_require_private_event_context(ldb);
path = lmdb_get_path(url);
if (path == NULL) {
@@ -836,16 +837,16 @@ int lmdb_connect(struct ldb_context *ldb,
return LDB_ERR_OPERATIONS_ERROR;
}
- ltdb = talloc_zero(ldb, struct ltdb_private);
- if (!ltdb) {
- ldb_oom(ldb);
- return LDB_ERR_OPERATIONS_ERROR;
- }
+ ltdb = talloc_zero(ldb, struct ltdb_private);
+ if (!ltdb) {
+ ldb_oom(ldb);
+ return LDB_ERR_OPERATIONS_ERROR;
+ }
lmdb = talloc_zero(ltdb, struct lmdb_private);
if (lmdb == NULL) {
TALLOC_FREE(ltdb);
- return ldb_oom(ldb);
+ return ldb_oom(ldb);
}
lmdb->ldb = ldb;
ltdb->kv_ops = &lmdb_key_value_ops;
@@ -869,6 +870,5 @@ int lmdb_connect(struct ldb_context *ldb,
*/
ltdb->max_key_length = LDB_MDB_MAX_KEY_LENGTH;
- return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
+ return init_store(ltdb, "ldb_mdb backend", ldb, options, _module);
}
-
diff --git a/lib/ldb/ldb_mdb/ldb_mdb.h b/lib/ldb/ldb_mdb/ldb_mdb.h
index 7e0729cc7af..8f21493927b 100644
--- a/lib/ldb/ldb_mdb/ldb_mdb.h
+++ b/lib/ldb/ldb_mdb/ldb_mdb.h
@@ -25,10 +25,8 @@
#ifndef _LDB_MDB_H_
#define _LDB_MDB_H_
-#include <lmdb.h>
-
#include "ldb_private.h"
-
+#include <lmdb.h>
struct lmdb_private {
struct ldb_context *ldb;
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 9d48137de38..4747deb8ca7 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -100,7 +100,18 @@ static int ltdb_lock_read(struct ldb_module *module)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
int tdb_ret = 0;
int ret;
-
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
if (tdb_transaction_active(ltdb->tdb) == false &&
ltdb->read_lock_count == 0) {
tdb_ret = tdb_lockall_read(ltdb->tdb);
@@ -128,6 +139,17 @@ static int ltdb_unlock_read(struct ldb_module *module)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
if (!tdb_transaction_active(ltdb->tdb) && ltdb->read_lock_count == 1) {
tdb_unlockall_read(ltdb->tdb);
ltdb->read_lock_count--;
@@ -653,11 +675,12 @@ static int ltdb_add(struct ltdb_context *ctx)
if (ltdb->max_key_length != 0 &&
ltdb->cache->GUID_index_attribute == NULL &&
- !ldb_dn_is_special(req->op.add.message->dn)) {
+ !ldb_dn_is_special(req->op.add.message->dn))
+ {
ldb_set_errstring(ldb_module_get_ctx(module),
"Must operate ldb_mdb in GUID "
"index mode, but " LTDB_IDXGUID " not set.");
- return LDB_ERR_UNWILLING_TO_PERFORM;
+ return LDB_ERR_UNWILLING_TO_PERFORM;
}
ret = ltdb_check_special_dn(module, req->op.add.message);
@@ -1446,21 +1469,69 @@ static int ltdb_rename(struct ltdb_context *ctx)
static int ltdb_tdb_transaction_start(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_start(ltdb->tdb);
}
static int ltdb_tdb_transaction_cancel(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_cancel(ltdb->tdb);
}
static int ltdb_tdb_transaction_prepare_commit(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_prepare_commit(ltdb->tdb);
}
static int ltdb_tdb_transaction_commit(struct ltdb_private *ltdb)
{
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
return tdb_transaction_commit(ltdb->tdb);
}
@@ -1469,6 +1540,18 @@ static int ltdb_start_trans(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(ltdb->module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
/* Do not take out the transaction lock on a read-only DB */
if (ltdb->read_only) {
return LDB_ERR_UNWILLING_TO_PERFORM;
@@ -1497,7 +1580,18 @@ static int ltdb_prepare_commit(struct ldb_module *module)
int ret;
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
-
+ pid_t pid = getpid();
+
+ if (ltdb->pid != pid) {
+ ldb_asprintf_errstring(
+ ldb_module_get_ctx(module),
+ __location__": Reusing ldb opend by pid %d in "
+ "process %d\n",
+ ltdb->pid,
+ pid);
+ return LDB_ERR_PROTOCOL_ERROR;
+ }
+
if (!ltdb->kv_ops->transaction_active(ltdb)) {
ldb_set_errstring(ldb_module_get_ctx(module),
"ltdb_prepare_commit() called "
@@ -2137,8 +2231,6 @@ int init_store(struct ltdb_private *ltdb,
const char *options[],
struct ldb_module **_module)
{
- struct ldb_module *module;
-
if (getenv("LDB_WARN_UNINDEXED")) {
ltdb->warn_unindexed = true;
}
@@ -2149,23 +2241,25 @@ int init_store(struct ltdb_private *ltdb,
ltdb->sequence_number = 0;
- module = ldb_module_new(ldb, ldb, name, <db_ops);
- if (!module) {
+ ltdb->pid = getpid();
+
+ ltdb->module = ldb_module_new(ldb, ldb, name, <db_ops);
+ if (!ltdb->module) {
ldb_oom(ldb);
talloc_free(ltdb);
return LDB_ERR_OPERATIONS_ERROR;
}
- ldb_module_set_private(module, ltdb);
- talloc_steal(module, ltdb);
+ ldb_module_set_private(ltdb->module, ltdb);
+ talloc_steal(ltdb->module, ltdb);
- if (ltdb_cache_load(module) != 0) {
+ if (ltdb_cache_load(ltdb->module) != 0) {
ldb_asprintf_errstring(ldb, "Unable to load ltdb cache "
"records for backend '%s'", name);
- talloc_free(module);
+ talloc_free(ltdb->module);
return LDB_ERR_OPERATIONS_ERROR;
}
- *_module = module;
+ *_module = ltdb->module;
/*
* Set or override the maximum key length
*
@@ -2261,6 +2355,7 @@ int ltdb_connect(struct ldb_context *ldb, const char *url,
ltdb->kv_ops = &key_value_ops;
+ errno = 0;
/* note that we use quite a large default hash size */
ltdb->tdb = ltdb_wrap_open(ltdb, path, 10000,
tdb_flags, open_flags,
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h
index 9c3f8d89d8d..f6819d7e325 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -36,6 +36,7 @@ struct kv_db_ops {
ldb_context */
struct ltdb_private {
const struct kv_db_ops *kv_ops;
+ struct ldb_module *module;
TDB_CONTEXT *tdb;
struct lmdb_private *lmdb_private;
unsigned int connect_flags;
@@ -76,6 +77,12 @@ struct ltdb_private {
* greater than this length will be rejected.
*/
unsigned max_key_length;
+
+ /*
+ * The PID that opened this database so we don't work in a
+ * fork()ed child.
+ */
+ pid_t pid;
};
struct ltdb_context {
diff --git a/lib/ldb/ldb_tdb/ldb_tdb_wrap.c b/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
index eb168098a75..4b94f820b59 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb_wrap.c
@@ -74,6 +74,7 @@ struct ltdb_wrap {
struct tdb_context *tdb;
dev_t device;
ino_t inode;
+ pid_t pid;
};
static struct ltdb_wrap *tdb_list;
@@ -105,9 +106,25 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
if (stat(path, &st) == 0) {
for (w=tdb_list;w;w=w->next) {
if (st.st_dev == w->device && st.st_ino == w->inode) {
+ pid_t pid = getpid();
+ int ret;
if (!talloc_reference(mem_ctx, w)) {
return NULL;
}
+ if (w->pid != pid) {
+ ret = tdb_reopen(w->tdb);
+ if (ret != 0) {
+ /*
+ * Avoid use-after-free:
+ * on fail the TDB
+ * is closed!
+ */
+ DLIST_REMOVE(tdb_list,
+ w);
+ return NULL;
+ }
+ w->pid = pid;
+ }
return w->tdb;
}
}
@@ -135,6 +152,7 @@ struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
w->device = st.st_dev;
w->inode = st.st_ino;
+ w->pid = getpid();
talloc_set_destructor(w, ltdb_wrap_destructor);
diff --git a/lib/ldb/tests/ldb_lmdb_size_test.c b/lib/ldb/tests/ldb_lmdb_size_test.c
index 1d897ec1717..af015fa72b5 100644
--- a/lib/ldb/tests/ldb_lmdb_size_test.c
+++ b/lib/ldb/tests/ldb_lmdb_size_test.c
@@ -49,10 +49,7 @@
#include <errno.h>
#include <unistd.h>
#include <talloc.h>
-
-#define TEVENT_DEPRECATED 1
#include <tevent.h>
-
#include <ldb.h>
#include <ldb_module.h>
#include <ldb_private.h>
@@ -61,6 +58,8 @@
#include <sys/wait.h>
+#include <lmdb.h>
+
#define TEST_BE "mdb"
diff --git a/lib/ldb/tests/ldb_lmdb_test.c b/lib/ldb/tests/ldb_lmdb_test.c
index c34be3cda6f..a254a849f4a 100644
--- a/lib/ldb/tests/ldb_lmdb_test.c
+++ b/lib/ldb/tests/ldb_lmdb_test.c
@@ -46,10 +46,7 @@
#include <errno.h>
#include <unistd.h>
#include <talloc.h>
-
-#define TEVENT_DEPRECATED 1
#include <tevent.h>
-
#include <ldb.h>
#include <ldb_module.h>
#include <ldb_private.h>
@@ -58,6 +55,8 @@
#include <sys/wait.h>
+#include "../ldb_tdb/ldb_tdb.h"
+#include "../ldb_mdb/ldb_mdb.h"
#define TEST_BE "mdb"
@@ -412,6 +411,142 @@ static void test_ldb_add_dn_no_guid_mode(void **state)
talloc_free(tmp_ctx);
}
+static struct MDB_env *get_mdb_env(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ struct lmdb_private *lmdb = NULL;
+ struct MDB_env *env = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ lmdb = ltdb->lmdb_private;
+ assert_non_null(lmdb);
+
+ env = lmdb->env;
+ assert_non_null(env);
+
+ return env;
+}
+
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ struct MDB_env *env3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+ env3 = get_mdb_env(ldb3);
+
+ assert_ptr_equal(env1, env2);
+ assert_ptr_equal(env1, env3);
+}
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct MDB_env *env1 = NULL;
+ struct MDB_env *env2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ env1 = get_mdb_env(ldb1);
+ env2 = get_mdb_env(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ struct MDB_env *env3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ env3 = get_mdb_env(ldb3);
+ if (env1 != env2) {
+ print_error(__location__": env1 != env2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (env1 == env3) {
+ print_error(__location__": env1 == env3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
int main(int argc, const char **argv)
{
const struct CMUnitTest tests[] = {
@@ -439,6 +574,14 @@ int main(int argc, const char **argv)
test_ldb_add_dn_no_guid_mode,
ldbtest_setup_noguid,
ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
};
return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c
index b8204685691..7b6f19c3a18 100644
--- a/lib/ldb/tests/ldb_mod_op_test.c
+++ b/lib/ldb/tests/ldb_mod_op_test.c
@@ -3985,73 +3985,11 @@ static void test_ldb_close_with_multiple_connections(void **state)
assert_int_equal(ret, 0);
}
-static struct MDB_env* get_mdb_env(struct ldb_context *ldb)
-{
- void *data = NULL;
- struct ltdb_private *ltdb = NULL;
- struct lmdb_private *lmdb = NULL;
- struct MDB_env *env = NULL;
-
- data = ldb_module_get_private(ldb->modules);
- assert_non_null(data);
-
- ltdb = talloc_get_type(data, struct ltdb_private);
- assert_non_null(ltdb);
-
- lmdb = ltdb->lmdb_private;
- assert_non_null(lmdb);
-
- env = lmdb->env;
- assert_non_null(env);
-
- return env;
-}
-static void test_multiple_opens(void **state)
-{
- struct ldb_context *ldb1 = NULL;
- struct ldb_context *ldb2 = NULL;
- struct ldb_context *ldb3 = NULL;
- struct MDB_env *env1 = NULL;
- struct MDB_env *env2 = NULL;
- struct MDB_env *env3 = NULL;
- int ret;
- struct ldbtest_ctx *test_ctx = NULL;
-
- test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
-
- /*
- * Open the database again
- */
- ldb1 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
- assert_int_equal(ret, 0);
-
- ldb2 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
- assert_int_equal(ret, 0);
-
- ldb3 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
- assert_int_equal(ret, 0);
- /*
- * We now have 3 ldb's open pointing to the same on disk database
- * they should all share the same MDB_env
- */
- env1 = get_mdb_env(ldb1);
- env2 = get_mdb_env(ldb2);
- env3 = get_mdb_env(ldb3);
-
- assert_ptr_equal(env1, env2);
- assert_ptr_equal(env1, env3);
-
-}
+#endif
-static void test_multiple_opens_across_fork(void **state)
+static void test_transaction_start_across_fork(void **state)
{
struct ldb_context *ldb1 = NULL;
- struct ldb_context *ldb2 = NULL;
- struct MDB_env *env1 = NULL;
- struct MDB_env *env2 = NULL;
int ret;
struct ldbtest_ctx *test_ctx = NULL;
int pipes[2];
@@ -4062,44 +4000,27 @@ static void test_multiple_opens_across_fork(void **state)
test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
/*
- * Open the database again
+ * Open the database
*/
ldb1 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
- assert_int_equal(ret, 0);
-
- ldb2 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
- env1 = get_mdb_env(ldb1);
- env2 = get_mdb_env(ldb2);
-
ret = pipe(pipes);
assert_int_equal(ret, 0);
child_pid = fork();
if (child_pid == 0) {
- struct ldb_context *ldb3 = NULL;
- struct MDB_env *env3 = NULL;
-
close(pipes[0]);
- ldb3 = ldb_init(test_ctx, test_ctx->ev);
- ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
- if (ret != 0) {
- print_error(__location__": ldb_connect returned (%d)\n",
- ret);
- exit(ret);
- }
- env3 = get_mdb_env(ldb3);
- if (env1 != env2) {
- print_error(__location__": env1 != env2\n");
- exit(LDB_ERR_OPERATIONS_ERROR);
- }
- if (env1 == env3) {
- print_error(__location__": env1 == env3\n");
- exit(LDB_ERR_OPERATIONS_ERROR);
+ ret = ldb_transaction_start(ldb1);
+ if (ret != LDB_ERR_PROTOCOL_ERROR) {
+ print_error(__location__": ldb_transaction_start "
+ "returned (%d) %s\n",
+ ret,
+ ldb1->err_string);
+ exit(LDB_ERR_OTHER);
}
+
ret = write(pipes[1], "GO", 2);
if (ret != 2) {
print_error(__location__
@@ -4121,7 +4042,7 @@ static void test_multiple_opens_across_fork(void **state)
assert_int_equal(WEXITSTATUS(wstatus), 0);
}
-static void test_transaction_start_across_fork(void **state)
+static void test_transaction_commit_across_fork(void **state)
{
struct ldb_context *ldb1 = NULL;
int ret;
@@ -4140,15 +4061,19 @@ static void test_transaction_start_across_fork(void **state)
ret = ldb_connect(ldb1, test_ctx->dbpath, 0, NULL);
assert_int_equal(ret, 0);
+ ret = ldb_transaction_start(ldb1);
+ assert_int_equal(ret, 0);
+
ret = pipe(pipes);
assert_int_equal(ret, 0);
child_pid = fork();
if (child_pid == 0) {
close(pipes[0]);
- ret = ldb_transaction_start(ldb1);
+ ret = ldb_transaction_commit(ldb1);
+
if (ret != LDB_ERR_PROTOCOL_ERROR) {
- print_error(__location__": ldb_transaction_start "
+ print_error(__location__": ldb_transaction_commit "
"returned (%d) %s\n",
ret,
ldb1->err_string);
@@ -4266,7 +4191,66 @@ static void test_lock_read_across_fork(void **state)
assert_int_equal(0, ret);
}
}
-#endif
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
int main(int argc, const char **argv)
{
@@ -4436,23 +4420,23 @@ int main(int argc, const char **argv)
test_ldb_close_with_multiple_connections,
ldb_search_test_setup,
ldb_search_test_teardown),
+#endif
cmocka_unit_test_setup_teardown(
- test_multiple_opens,
+ test_transaction_start_across_fork,
ldbtest_setup,
ldbtest_teardown),
cmocka_unit_test_setup_teardown(
- test_multiple_opens_across_fork,
+ test_transaction_commit_across_fork,
ldbtest_setup,
ldbtest_teardown),
cmocka_unit_test_setup_teardown(
- test_transaction_start_across_fork,
+ test_lock_read_across_fork,
ldbtest_setup,
ldbtest_teardown),
cmocka_unit_test_setup_teardown(
- test_lock_read_across_fork,
+ test_multiple_opens_across_fork,
ldbtest_setup,
ldbtest_teardown),
-#endif
};
return cmocka_run_group_tests(tests, NULL, NULL);
diff --git a/lib/ldb/tests/ldb_tdb_test.c b/lib/ldb/tests/ldb_tdb_test.c
new file mode 100644
index 00000000000..abba8cb8f69
--- /dev/null
+++ b/lib/ldb/tests/ldb_tdb_test.c
@@ -0,0 +1,387 @@
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Copyright (C) Andrew Bartlett <abartlet at samba.org> 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * lmdb backend specific tests for ldb
+ *
+ * Setup and tear down code copied from ldb_mod_op_test.c
+ */
+
+/*
+ * from cmocka.c:
+ * These headers or their equivalents should be included prior to
+ * including
+ * this header file.
+ *
+ * #include <stdarg.h>
+ * #include <stddef.h>
+ * #include <setjmp.h>
+ *
+ * This allows test applications to use custom definitions of C standard
+ * library functions and types.
+ *
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <errno.h>
+#include <unistd.h>
+#include <talloc.h>
+#include <tevent.h>
+#include <ldb.h>
+#include <ldb_module.h>
+#include <ldb_private.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <sys/wait.h>
+
+#include "../ldb_tdb/ldb_tdb.h"
+
+#define TEST_BE "tdb"
+
+struct ldbtest_ctx {
+ struct tevent_context *ev;
+ struct ldb_context *ldb;
+
+ const char *dbfile;
+
+ const char *dbpath;
+};
+
+static void unlink_old_db(struct ldbtest_ctx *test_ctx)
+{
+ int ret;
+
+ errno = 0;
+ ret = unlink(test_ctx->dbfile);
+ if (ret == -1 && errno != ENOENT) {
+ fail();
+ }
+}
+
+static int ldbtest_noconn_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+
+ test_ctx = talloc_zero(NULL, struct ldbtest_ctx);
+ assert_non_null(test_ctx);
+
+ test_ctx->ev = tevent_context_init(test_ctx);
+ assert_non_null(test_ctx->ev);
+
+ test_ctx->ldb = ldb_init(test_ctx, test_ctx->ev);
+ assert_non_null(test_ctx->ldb);
+
+ test_ctx->dbfile = talloc_strdup(test_ctx, "apitest.ldb");
+ assert_non_null(test_ctx->dbfile);
+
+ test_ctx->dbpath = talloc_asprintf(test_ctx,
+ TEST_BE"://%s", test_ctx->dbfile);
+ assert_non_null(test_ctx->dbpath);
+
+ unlink_old_db(test_ctx);
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_noconn_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+
+ unlink_old_db(test_ctx);
+ talloc_free(test_ctx);
+ return 0;
+}
+
+static int ldbtest_setup(void **state)
+{
+ struct ldbtest_ctx *test_ctx;
+ int ret;
+ struct ldb_ldif *ldif;
+ const char *index_ldif = \
+ "dn: @INDEXLIST\n"
+ "@IDXGUID: objectUUID\n"
+ "@IDX_DN_GUID: GUID\n"
+ "\n";
+
+ ldbtest_noconn_setup((void **) &test_ctx);
+
+ ret = ldb_connect(test_ctx->ldb, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+
+ while ((ldif = ldb_ldif_read_string(test_ctx->ldb, &index_ldif))) {
+ ret = ldb_add(test_ctx->ldb, ldif->msg);
+ assert_int_equal(ret, LDB_SUCCESS);
+ }
+ *state = test_ctx;
+ return 0;
+}
+
+static int ldbtest_teardown(void **state)
+{
+ struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state,
+ struct ldbtest_ctx);
+ ldbtest_noconn_teardown((void **) &test_ctx);
+ return 0;
+}
+
+
+static TDB_CONTEXT *get_tdb_context(struct ldb_context *ldb)
+{
+ void *data = NULL;
+ struct ltdb_private *ltdb = NULL;
+ TDB_CONTEXT *tdb = NULL;
+
+ data = ldb_module_get_private(ldb->modules);
+ assert_non_null(data);
+
+ ltdb = talloc_get_type(data, struct ltdb_private);
+ assert_non_null(ltdb);
+
+ tdb = ltdb->tdb;
+ assert_non_null(tdb);
+
+ return tdb;
+}
+
+static void test_multiple_opens(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ struct ldb_context *ldb3 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ TDB_CONTEXT *tdb3 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ assert_int_equal(ret, 0);
+ /*
+ * We now have 3 ldb's open pointing to the same on disk database
+ * they should all share the same MDB_env
+ */
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+ tdb3 = get_tdb_context(ldb3);
+
+ assert_ptr_equal(tdb1, tdb2);
+ assert_ptr_equal(tdb1, tdb3);
+}
+
+static void test_multiple_opens_across_fork(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+ TDB_CONTEXT *tdb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret != 0) {
+ print_error(__location__": ldb_connect returned (%d)\n",
+ ret);
+ exit(ret);
+ }
+ tdb3 = get_tdb_context(ldb3);
+ if (tdb1 != tdb2) {
+ print_error(__location__": tdb1 != tdb2\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ if (tdb1 != tdb3) {
+ print_error(__location__": tdb1 != tdb3\n");
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+static void test_multiple_opens_across_fork_triggers_reopen(void **state)
+{
+ struct ldb_context *ldb1 = NULL;
+ struct ldb_context *ldb2 = NULL;
+ TDB_CONTEXT *tdb1 = NULL;
+ TDB_CONTEXT *tdb2 = NULL;
+ int ret;
+ struct ldbtest_ctx *test_ctx = NULL;
+ int pipes[2];
+ char buf[2];
+ int wstatus;
+ pid_t pid, child_pid;
+
+ test_ctx = talloc_get_type_abort(*state, struct ldbtest_ctx);
+
+ /*
+ * Open the database again
+ */
+ ldb1 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb1, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ ldb2 = ldb_init(test_ctx, test_ctx->ev);
+ ret = ldb_connect(ldb2, test_ctx->dbpath, LDB_FLG_RDONLY, NULL);
+ assert_int_equal(ret, 0);
+
+ tdb1 = get_tdb_context(ldb1);
+ tdb2 = get_tdb_context(ldb2);
+ assert_ptr_equal(tdb1, tdb2);
+
+ /*
+ * Break the internal tdb_reopen() by making a
+ * transaction
+ *
+ * This shows that the tdb_reopen() is called, which is
+ * essential if the host OS does not have pread()
+ */
+ ret = tdb_transaction_start(tdb1);
+ assert_int_equal(ret, 0);
+
+ ret = pipe(pipes);
+ assert_int_equal(ret, 0);
+
+ child_pid = fork();
+ if (child_pid == 0) {
+ struct ldb_context *ldb3 = NULL;
+
+ close(pipes[0]);
+ ldb3 = ldb_init(test_ctx, test_ctx->ev);
+
+ /*
+ * This should fail as we have taken out a lock
+ * against the raw TDB above, and tdb_reopen()
+ * will fail in that state.
+ *
+ * This check matters as tdb_reopen() is important
+ * if the host does not have pread()
+ */
+ ret = ldb_connect(ldb3, test_ctx->dbpath, 0, NULL);
+ if (ret == 0) {
+ print_error(__location__": ldb_connect expected "
+ "LDB_ERR_OPERATIONS_ERROR "
+ "returned (%d)\n",
+ ret);
+ exit(5000);
+ }
+ ret = write(pipes[1], "GO", 2);
+ if (ret != 2) {
+ print_error(__location__
+ " write returned (%d)",
+ ret);
+ exit(LDB_ERR_OPERATIONS_ERROR);
+ }
+ exit(LDB_SUCCESS);
+ }
+ close(pipes[1]);
+ ret = read(pipes[0], buf, 2);
+ assert_int_equal(ret, 2);
+
+ pid = waitpid(child_pid, &wstatus, 0);
+ assert_int_equal(pid, child_pid);
+
+ assert_true(WIFEXITED(wstatus));
+
+ assert_int_equal(WEXITSTATUS(wstatus), 0);
+}
+
+int main(int argc, const char **argv)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork,
+ ldbtest_setup,
+ ldbtest_teardown),
+ cmocka_unit_test_setup_teardown(
+ test_multiple_opens_across_fork_triggers_reopen,
+ ldbtest_setup,
+ ldbtest_teardown),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}
diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py
index 1222d122199..72fc0d9624e 100755
--- a/lib/ldb/tests/python/api.py
+++ b/lib/ldb/tests/python/api.py
@@ -15,6 +15,12 @@ PY3 = sys.version_info > (3, 0)
TDB_PREFIX = "tdb://"
MDB_PREFIX = "mdb://"
+MDB_INDEX_OBJ = {
+ "dn": "@INDEXLIST",
+ "@IDXONE": [b"1"],
+ "@IDXGUID": [b"objectUUID"],
+ "@IDX_DN_GUID": [b"GUID"]
+}
def tempdir():
import tempfile
@@ -670,10 +676,7 @@ class SimpleLdbLmdb(SimpleLdb):
def setUp(self):
self.prefix = MDB_PREFIX
- self.index = {"dn": "@INDEXLIST",
- "@IDXONE": [b"1"],
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
+ self.index = MDB_INDEX_OBJ
super(SimpleLdbLmdb, self).setUp()
def tearDown(self):
@@ -1112,9 +1115,7 @@ class SearchTestsLmdb(SearchTests):
def setUp(self):
self.prefix = MDB_PREFIX
- self.index = {"dn": "@INDEXLIST",
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
+ self.index = MDB_INDEX_OBJ
super(SearchTestsLmdb, self).setUp()
def tearDown(self):
@@ -1404,6 +1405,16 @@ class AddModifyTests(LdbBaseTest):
"objectUUID": b"0123456789abcde3"})
+class AddModifyTestsLmdb(AddModifyTests):
+
+ def setUp(self):
+ self.prefix = MDB_PREFIX
+ self.index = MDB_INDEX_OBJ
+ super(AddModifyTestsLmdb, self).setUp()
+
+ def tearDown(self):
+ super(AddModifyTestsLmdb, self).tearDown()
+
class IndexedAddModifyTests(AddModifyTests):
"""Test searches using the index, to ensure the index doesn't
break things"""
@@ -2575,9 +2586,7 @@ class LdbResultTestsLmdb(LdbResultTests):
def setUp(self):
self.prefix = MDB_PREFIX
- self.index = {"dn": "@INDEXLIST",
- "@IDXGUID": [b"objectUUID"],
- "@IDX_DN_GUID": [b"GUID"]}
+ self.index = MDB_INDEX_OBJ
super(LdbResultTestsLmdb, self).setUp()
def tearDown(self):
diff --git a/lib/ldb/tools/ldbdump.c b/lib/ldb/tools/ldbdump.c
index 2da2ca8ec70..4697661a59d 100644
--- a/lib/ldb/tools/ldbdump.c
+++ b/lib/ldb/tools/ldbdump.c
@@ -28,7 +28,7 @@
#include <ldb_private.h>
#ifdef HAVE_LMDB
-#include "lmdb.h"
+#include <lmdb.h>
#endif /* ifdef HAVE_LMDB */
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index cbb7fcc9c4f..46472db0e6b 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -380,6 +380,7 @@ def build(bld):
else:
lmdb_deps = ''
+
bld.SAMBA_MODULE('ldb_ldb',
bld.SUBDIR('ldb_ldb',
'''ldb_ldb.c'''),
@@ -439,11 +440,16 @@ def build(bld):
deps='cmocka ldb',
install=False)
+ bld.SAMBA_BINARY('ldb_tdb_test',
+ source='tests/ldb_tdb_test.c',
+ deps='cmocka ldb',
+ install=False)
+
if bld.CONFIG_SET('HAVE_LMDB'):
bld.SAMBA_BINARY('ldb_mdb_mod_op_test',
source='tests/ldb_mod_op_test.c',
- cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 ' +
- '-DTEST_LMDB=1',
+ cflags='-DTEST_BE=\"mdb\" -DGUID_IDX=1 '
+ + '-DTEST_LMDB=1',
deps='cmocka ldb lmdb',
install=False)
@@ -509,6 +515,9 @@ def test(ctx):
if env.ENABLE_MDB_BACKEND:
test_exes.append('ldb_mdb_mod_op_test')
test_exes.append('ldb_lmdb_test')
+ # we don't want to run ldb_lmdb_size_test (which proves we can
+ # fit > 4G of data into the DB), it would fill up the disk on
+ # many of our test instances
test_exes.append('ldb_mdb_kv_ops_test')
for test_exe in test_exes:
cmd = os.path.join(Utils.g_module.blddir, test_exe)
More information about the samba-technical
mailing list