[PATCH] Re: Proposed ldb 1.1.30 and tdb 1.3.13 (improve AD DC search performance, make multi-process)
Stefan Metzmacher
metze at samba.org
Tue Apr 11 16:08:27 UTC 2017
Am 05.04.2017 um 07:06 schrieb Andrew Bartlett via samba-technical:
> On Wed, 2017-04-05 at 12:19 +1200, Andrew Bartlett wrote:
>>
>> Attached is the whole set of patches I'm currently trying to land, as
>> well as just the set discussed. I've improved the whitespace.
>
> Attached is an updated patch of the whole series, to avoid confusion.
Here's an update on current master.
I've added the run-fcntl-deadlock as test.
I still want to run the standalone make test of tdb and ldb on solaris,
before we push this and have a closer look at the ldb changes.
metze
-------------- next part --------------
From 604a4363810457633c6c2239e1df0f4c7a28f254 Mon Sep 17 00:00:00 2001
From: Ralph Boehme <slow at samba.org>
Date: Tue, 14 Mar 2017 14:24:18 +0100
Subject: [PATCH 01/17] tdb: runtime check for robust mutexes may hang in
threaded programs
The current runtime check for robust mutexes in
tdb_runtime_check_for_robust_mutexes() is not thread-safe.
When called in a multi-threaded program where any another thread doesn't
have SIGCHLD blocked, we may end up hung in sigsuspend() waiting for a
SIGCHLD of a child procecss and the signal was delivered to another
thread.
Revert to the previous behaviour of waiting for the child instead of
waiting for the SIGCHLD signal.
Ensure the pid we wait for is not reset to -1 in a toctou race with the
signal handler.
Check whether waitpid() returns ECHILD which can happen if the signal
handler is run by more then one thread in parallel (yes, this can
happen) or if tdb_robust_mutex_wait_for_child() and the signal handler
are racing.
Bug: https://bugzilla.samba.org/show_bug.cgi?id=12593
Pair-programmed-with: Stefan Metzmacher <metze at samba.org>
Signed-off-by: Ralph Boehme <slow at samba.org>
Signed-off-by: Stefan Metzmacher <metze at samba.org>
---
lib/tdb/common/mutex.c | 116 +++++++++++++++++++++++++++++--------------------
1 file changed, 70 insertions(+), 46 deletions(-)
diff --git a/lib/tdb/common/mutex.c b/lib/tdb/common/mutex.c
index cac3916..8a122d5 100644
--- a/lib/tdb/common/mutex.c
+++ b/lib/tdb/common/mutex.c
@@ -752,12 +752,23 @@ static bool tdb_robust_mutex_setup_sigchild(void (*handler)(int),
static void tdb_robust_mutex_handler(int sig)
{
- if (tdb_robust_mutex_pid != -1) {
+ pid_t child_pid = tdb_robust_mutex_pid;
+
+ if (child_pid != -1) {
pid_t pid;
- int status;
- pid = waitpid(tdb_robust_mutex_pid, &status, WNOHANG);
- if (pid == tdb_robust_mutex_pid) {
+ pid = waitpid(child_pid, NULL, WNOHANG);
+ if (pid == -1) {
+ switch (errno) {
+ case ECHILD:
+ tdb_robust_mutex_pid = -1;
+ return;
+
+ default:
+ return;
+ }
+ }
+ if (pid == child_pid) {
tdb_robust_mutex_pid = -1;
return;
}
@@ -776,6 +787,44 @@ static void tdb_robust_mutex_handler(int sig)
tdb_robust_mutext_old_handler(sig);
}
+static void tdb_robust_mutex_wait_for_child(pid_t *child_pid)
+{
+ int options = WNOHANG;
+
+ if (*child_pid == -1) {
+ return;
+ }
+
+ while (tdb_robust_mutex_pid > 0) {
+ pid_t pid;
+
+ /*
+ * First we try with WNOHANG, as the process might not exist
+ * anymore. Once we've sent SIGKILL we block waiting for the
+ * exit.
+ */
+ pid = waitpid(*child_pid, NULL, options);
+ if (pid == -1) {
+ if (errno == EINTR) {
+ continue;
+ } else if (errno == ECHILD) {
+ break;
+ } else {
+ abort();
+ }
+ }
+ if (pid == *child_pid) {
+ break;
+ }
+
+ kill(*child_pid, SIGKILL);
+ options = 0;
+ }
+
+ tdb_robust_mutex_pid = -1;
+ *child_pid = -1;
+}
+
_PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
{
void *ptr = NULL;
@@ -788,9 +837,8 @@ _PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
char c = 0;
bool ok;
static bool initialized;
- sigset_t mask, old_mask, suspend_mask;
+ pid_t saved_child_pid = -1;
bool cleanup_ma = false;
- bool cleanup_sigmask = false;
if (initialized) {
return tdb_mutex_locking_cached;
@@ -798,8 +846,6 @@ _PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
initialized = true;
- sigemptyset(&suspend_mask);
-
ok = tdb_mutex_locking_supported();
if (!ok) {
return false;
@@ -845,26 +891,13 @@ _PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
}
m = (pthread_mutex_t *)ptr;
- /*
- * Block SIGCHLD so we can atomically wait for it later with
- * sigsuspend()
- */
- sigemptyset(&mask);
- sigaddset(&mask, SIGCHLD);
- ret = pthread_sigmask(SIG_BLOCK, &mask, &old_mask);
- if (ret != 0) {
- goto cleanup;
- }
- cleanup_sigmask = true;
- suspend_mask = old_mask;
- sigdelset(&suspend_mask, SIGCHLD);
-
if (tdb_robust_mutex_setup_sigchild(tdb_robust_mutex_handler,
&tdb_robust_mutext_old_handler) == false) {
goto cleanup;
}
tdb_robust_mutex_pid = fork();
+ saved_child_pid = tdb_robust_mutex_pid;
if (tdb_robust_mutex_pid == 0) {
size_t nwritten;
close(pipe_down[1]);
@@ -914,14 +947,7 @@ _PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
goto cleanup;
}
- while (tdb_robust_mutex_pid > 0) {
- ret = sigsuspend(&suspend_mask);
- if (ret != -1 || errno != EINTR) {
- abort();
- }
- }
- tdb_robust_mutex_setup_sigchild(tdb_robust_mutext_old_handler, NULL);
- tdb_robust_mutext_old_handler = SIG_ERR;
+ tdb_robust_mutex_wait_for_child(&saved_child_pid);
ret = pthread_mutex_trylock(m);
if (ret != EOWNERDEAD) {
@@ -950,23 +976,21 @@ _PUBLIC_ bool tdb_runtime_check_for_robust_mutexes(void)
tdb_mutex_locking_cached = true;
cleanup:
- while (tdb_robust_mutex_pid > 0) {
- kill(tdb_robust_mutex_pid, SIGKILL);
- ret = sigsuspend(&suspend_mask);
- if (ret != -1 || errno != EINTR) {
- abort();
- }
- }
+ /*
+ * Note that we don't reset the signal handler we just reset
+ * tdb_robust_mutex_pid to -1. This is ok as this code path is only
+ * called once per process.
+ *
+ * Leaving our signal handler avoids races with other threads potentialy
+ * setting up their SIGCHLD handlers.
+ *
+ * The worst thing that can happen is that the other newer signal
+ * handler will get the SIGCHLD signal for our child and/or reap the
+ * child with a wait() function. tdb_robust_mutex_wait_for_child()
+ * handles the case where waitpid returns ECHILD.
+ */
+ tdb_robust_mutex_wait_for_child(&saved_child_pid);
- if (tdb_robust_mutext_old_handler != SIG_ERR) {
- tdb_robust_mutex_setup_sigchild(tdb_robust_mutext_old_handler, NULL);
- }
- if (cleanup_sigmask) {
- ret = pthread_sigmask(SIG_SETMASK, &old_mask, NULL);
- if (ret != 0) {
- abort();
- }
- }
if (m != NULL) {
pthread_mutex_destroy(m);
}
--
1.9.1
From 86b540cb11a8b27c474fa1468f03efdb70510ca8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 19:11:06 +1300
Subject: [PATCH 02/17] tdb: Improve debugging when the allrecord lock fails to
upgrade
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/tdb/common/lock.c | 2 ++
lib/tdb/common/transaction.c | 18 +++++++++++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/lib/tdb/common/lock.c b/lib/tdb/common/lock.c
index 4ad70cf..e330201 100644
--- a/lib/tdb/common/lock.c
+++ b/lib/tdb/common/lock.c
@@ -257,12 +257,14 @@ int tdb_allrecord_upgrade(struct tdb_context *tdb)
TDB_LOG((tdb, TDB_DEBUG_ERROR,
"tdb_allrecord_upgrade failed: count %u too high\n",
tdb->allrecord_lock.count));
+ tdb->ecode = TDB_ERR_LOCK;
return -1;
}
if (tdb->allrecord_lock.off != 1) {
TDB_LOG((tdb, TDB_DEBUG_ERROR,
"tdb_allrecord_upgrade failed: already upgraded?\n"));
+ tdb->ecode = TDB_ERR_LOCK;
return -1;
}
diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
index 0dd057b..e22aada 100644
--- a/lib/tdb/common/transaction.c
+++ b/lib/tdb/common/transaction.c
@@ -982,7 +982,23 @@ static int _tdb_transaction_prepare_commit(struct tdb_context *tdb)
/* upgrade the main transaction lock region to a write lock */
if (tdb_allrecord_upgrade(tdb) == -1) {
- TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_transaction_prepare_commit: failed to upgrade hash locks\n"));
+ if (tdb->ecode == TDB_ERR_RDONLY && tdb->read_only) {
+ TDB_LOG((tdb, TDB_DEBUG_ERROR,
+ "tdb_transaction_prepare_commit: "
+ "failed to upgrade hash locks: "
+ "database is read only\n"));
+ } else if (tdb->ecode == TDB_ERR_RDONLY
+ && tdb->traverse_read) {
+ TDB_LOG((tdb, TDB_DEBUG_ERROR,
+ "tdb_transaction_prepare_commit: "
+ "failed to upgrade hash locks: "
+ "a database traverse is in progress\n"));
+ } else {
+ TDB_LOG((tdb, TDB_DEBUG_ERROR,
+ "tdb_transaction_prepare_commit: "
+ "failed to upgrade hash locks: %s\n",
+ tdb_errorstr(tdb)));
+ }
_tdb_transaction_cancel(tdb);
return -1;
}
--
1.9.1
From fe5908f10d9e73450414bba2db9d6070224fa371 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 31 Mar 2017 17:35:06 +1300
Subject: [PATCH 03/17] tdb: Improve debugging in _tdb_transaction_start
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/tdb/common/transaction.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
index e22aada..420e754 100644
--- a/lib/tdb/common/transaction.c
+++ b/lib/tdb/common/transaction.c
@@ -476,6 +476,10 @@ static int _tdb_transaction_start(struct tdb_context *tdb,
SAFE_FREE(tdb->transaction);
if ((lockflags & TDB_LOCK_WAIT) == 0) {
tdb->ecode = TDB_ERR_NOLOCK;
+ } else {
+ TDB_LOG((tdb, TDB_DEBUG_ERROR,
+ "tdb_transaction_start: "
+ "failed to get transaction lock\n"));
}
return -1;
}
--
1.9.1
From cdbd16a1bf38b1f2ab8433a3c680b1fd452e03e2 Mon Sep 17 00:00:00 2001
From: Stefan Metzmacher <metze at samba.org>
Date: Tue, 11 Apr 2017 17:21:20 +0200
Subject: [PATCH 04/17] tdb: add run-fcntl-deadlock test
This verifies the F_RDLCK => F_WRLCK upgrade logic in the kernel
for conflicting locks.
This is a standalone test to check the traverse_read vs.
allrecord_lock/prepare_commit interaction.
This is based on the example from
https://lists.samba.org/archive/samba-technical/2017-April/119861.html
from Douglas Bagnall <douglas.bagnall at catalyst.net.nz> and Volker Lendecke <vl at samba.org>.
Signed-off-by: Stefan Metzmacher <metze at samba.org>
---
lib/tdb/test/run-fcntl-deadlock.c | 202 ++++++++++++++++++++++++++++++++++++++
lib/tdb/wscript | 1 +
2 files changed, 203 insertions(+)
create mode 100644 lib/tdb/test/run-fcntl-deadlock.c
diff --git a/lib/tdb/test/run-fcntl-deadlock.c b/lib/tdb/test/run-fcntl-deadlock.c
new file mode 100644
index 0000000..0a328af
--- /dev/null
+++ b/lib/tdb/test/run-fcntl-deadlock.c
@@ -0,0 +1,202 @@
+#include "../common/tdb_private.h"
+#include "../common/io.c"
+#include "../common/tdb.c"
+#include "../common/lock.c"
+#include "../common/freelist.c"
+#include "../common/traverse.c"
+#include "../common/transaction.c"
+#include "../common/error.c"
+#include "../common/open.c"
+#include "../common/check.c"
+#include "../common/hash.c"
+#include "../common/mutex.c"
+#include "replace.h"
+#include "system/filesys.h"
+#include "system/time.h"
+#include <errno.h>
+#include "tap-interface.h"
+
+/*
+ * This tests the low level locking requirement
+ * for the allrecord lock/prepare_commit and traverse_read interaction.
+ *
+ * The pattern with the traverse_read and prepare_commit interaction is
+ * the following:
+ *
+ * 1. transaction_start got the allrecord lock with F_RDLCK.
+ *
+ * 2. the traverse_read code walks the database in a sequence like this
+ * (per chain):
+ * 2.1 chainlock(chainX, F_RDLCK)
+ * 2.2 recordlock(chainX.record1, F_RDLCK)
+ * 2.3 chainunlock(chainX, F_RDLCK)
+ * 2.4 callback(chainX.record1)
+ * 2.5 chainlock(chainX, F_RDLCK)
+ * 2.6 recordunlock(chainX.record1, F_RDLCK)
+ * 2.7 recordlock(chainX.record2, F_RDLCK)
+ * 2.8 chainunlock(chainX, F_RDLCK)
+ * 2.9 callback(chainX.record2)
+ * 2.10 chainlock(chainX, F_RDLCK)
+ * 2.11 recordunlock(chainX.record2, F_RDLCK)
+ * 2.12 chainunlock(chainX, F_RDLCK)
+ * 2.13 goto next chain
+ *
+ * So it has always one record locked in F_RDLCK mode and tries to
+ * get the 2nd one before it releases the first one.
+ *
+ * 3. prepare_commit tries to upgrade the allrecord lock to F_RWLCK
+ * If that happens at the time of 2.4, the operation of
+ * 2.5 may deadlock with the allrecord lock upgrade.
+ * On Linux step 2.5 works in order to make some progress with the
+ * locking, but on solaris it might fail because the kernel
+ * wants to satisfy the 1st lock requester before the 2nd one.
+ *
+ * I think the first step is a standalone test that does this:
+ *
+ * process1: F_RDLCK for ofs=0 len=2
+ * process2: F_RDLCK for ofs=0 len=1
+ * process1: upgrade ofs=0 len=2 to F_RWLCK (in blocking mode)
+ * process2: F_RDLCK for ofs=1 len=1
+ * process2: unlock ofs=0 len=2
+ * process1: should continue at that point
+ *
+ * Such a test follows here...
+ */
+
+static int raw_fcntl_lock(int fd, int rw, off_t off, off_t len, bool waitflag)
+{
+ struct flock fl;
+ int cmd;
+ fl.l_type = rw;
+ fl.l_whence = SEEK_SET;
+ fl.l_start = off;
+ fl.l_len = len;
+ fl.l_pid = 0;
+
+ cmd = waitflag ? F_SETLKW : F_SETLK;
+
+ return fcntl(fd, cmd, &fl);
+}
+
+static int raw_fcntl_unlock(int fd, off_t off, off_t len)
+{
+ struct flock fl;
+ fl.l_type = F_UNLCK;
+ fl.l_whence = SEEK_SET;
+ fl.l_start = off;
+ fl.l_len = len;
+ fl.l_pid = 0;
+
+ return fcntl(fd, F_SETLKW, &fl);
+}
+
+
+int pipe_r;
+int pipe_w;
+char buf[2];
+
+static void expect_char(char c)
+{
+ read(pipe_r, buf, 1);
+ if (*buf != c) {
+ fail("We were expecting %c, but got %c", c, buf[0]);
+ }
+}
+
+static void send_char(char c)
+{
+ write(pipe_w, &c, 1);
+}
+
+
+int main(int argc, char *argv[])
+{
+ int process;
+ int fd;
+ const char *filename = "run-fcntl-deadlock.lck";
+ int pid;
+ int pipes_1_2[2];
+ int pipes_2_1[2];
+ int ret;
+
+ pipe(pipes_1_2);
+ pipe(pipes_2_1);
+ fd = open(filename, O_RDWR | O_CREAT, 0755);
+
+ pid = fork();
+ if (pid == 0) {
+ pipe_r = pipes_1_2[0];
+ pipe_w = pipes_2_1[1];
+ process = 2;
+ alarm(15);
+ } else {
+ pipe_r = pipes_2_1[0];
+ pipe_w = pipes_1_2[1];
+ process = 1;
+ alarm(15);
+ }
+
+ /* a: process1: F_RDLCK for ofs=0 len=2 */
+ if (process == 1) {
+ ret = raw_fcntl_lock(fd, F_RDLCK, 0, 2, true);
+ ok(ret == 0,
+ "process 1 lock ofs=0 len=2: %d - %s",
+ ret, strerror(errno));
+ diag("process 1 took read lock on range 0,2");
+ send_char('a');
+ }
+
+ /* process2: F_RDLCK for ofs=0 len=1 */
+ if (process == 2) {
+ expect_char('a');
+ ret = raw_fcntl_lock(fd, F_RDLCK, 0, 1, true);
+ ok(ret == 0,
+ "process 2 lock ofs=0 len=1: %d - %s",
+ ret, strerror(errno));;
+ diag("process 2 took read lock on range 0,1");
+ send_char('b');
+ }
+
+ /* process1: upgrade ofs=0 len=2 to F_RWLCK (in blocking mode) */
+ if (process == 1) {
+ expect_char('b');
+ send_char('c');
+ diag("process 1 starts upgrade on range 0,2");
+ ret = raw_fcntl_lock(fd, F_WRLCK, 0, 2, true);
+ ok(ret == 0,
+ "process 1 RW lock ofs=0 len=2: %d - %s",
+ ret, strerror(errno));
+ diag("process 1 got read upgrade done");
+ /* at this point process 1 is blocked on 2 releasing the
+ read lock */
+ }
+
+ /*
+ * process2: F_RDLCK for ofs=1 len=1
+ * process2: unlock ofs=0 len=2
+ */
+ if (process == 2) {
+ expect_char('c'); /* we know process 1 is *about* to lock */
+ sleep(1);
+ ret = raw_fcntl_lock(fd, F_RDLCK, 1, 1, true);
+ ok(ret == 0,
+ "process 2 lock ofs=1 len=1: %d - %s",
+ ret, strerror(errno));
+ diag("process 2 got read lock on 1,1\n");
+ ret = raw_fcntl_unlock(fd, 0, 2);
+ ok(ret == 0,
+ "process 2 unlock ofs=0 len=2: %d - %s",
+ ret, strerror(errno));
+ diag("process 2 released read lock on 0,2\n");
+ sleep(1);
+ send_char('d');
+ }
+
+ if (process == 1) {
+ expect_char('d');
+ }
+
+ diag("process %d has got to the end\n", process);
+
+ return 0;
+}
diff --git a/lib/tdb/wscript b/lib/tdb/wscript
index 693787c..79b0e6f 100644
--- a/lib/tdb/wscript
+++ b/lib/tdb/wscript
@@ -41,6 +41,7 @@ tdb1_unit_tests = [
'run-traverse-in-transaction',
'run-wronghash-fail',
'run-zero-append',
+ 'run-fcntl-deadlock',
'run-marklock-deadlock',
'run-allrecord-traverse-deadlock',
'run-mutex-openflags2',
--
1.9.1
From b0a42f3da8cb8e2856cfbaa3c8ff47295b44e62d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Fri, 31 Mar 2017 17:34:13 +1300
Subject: [PATCH 05/17] tdb: Remove locking from tdb_traverse_read()
This restores the original intent of tdb_traverse_read() in
7dd31288a701d772e45b1960ac4ce4cc1be782ed
This is needed to avoid a deadlock with tdb_lockall() and the
transaction start, as ldb_tdb should take the allrecord lock during a
search (which calls tdb_traverse), and can otherwise deadlock against
a transaction starting in another process
We add a test to show that a transaction can now start while a read
traverse is in progress
This allows more operations to happen in parallel. The blocking point
is moved to the prepare commit.
This in turn permits a roughly doubling of unindexed search
performance, because currently ldb_tdb omits to take the lock due to
an unrelated bug, but taking the allrecord lock triggers the
above-mentioned deadlock.
This behaviour was added in 251aaafe3a9213118ac3a92def9ab2104c40d12a for
Solaris 10 in 2005. But the run-fcntl-deadlock test works also on Solaris 10,
see https://lists.samba.org/archive/samba-technical/2017-April/119876.html.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/tdb/common/traverse.c | 10 +---------
lib/tdb/test/run-nested-traverse.c | 31 +++++++++++++++++++++++++++----
2 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/lib/tdb/common/traverse.c b/lib/tdb/common/traverse.c
index f33ef34..f62306e 100644
--- a/lib/tdb/common/traverse.c
+++ b/lib/tdb/common/traverse.c
@@ -244,7 +244,7 @@ out:
/*
- a read style traverse - temporarily marks the db read only
+ a read style traverse - temporarily marks each record read only
*/
_PUBLIC_ int tdb_traverse_read(struct tdb_context *tdb,
tdb_traverse_func fn, void *private_data)
@@ -252,19 +252,11 @@ _PUBLIC_ int tdb_traverse_read(struct tdb_context *tdb,
struct tdb_traverse_lock tl = { NULL, 0, 0, F_RDLCK };
int ret;
- /* we need to get a read lock on the transaction lock here to
- cope with the lock ordering semantics of solaris10 */
- if (tdb_transaction_lock(tdb, F_RDLCK, TDB_LOCK_WAIT)) {
- return -1;
- }
-
tdb->traverse_read++;
tdb_trace(tdb, "tdb_traverse_read_start");
ret = tdb_traverse_internal(tdb, fn, private_data, &tl);
tdb->traverse_read--;
- tdb_transaction_unlock(tdb, F_RDLCK);
-
return ret;
}
diff --git a/lib/tdb/test/run-nested-traverse.c b/lib/tdb/test/run-nested-traverse.c
index 22ee3e2..aeaa085 100644
--- a/lib/tdb/test/run-nested-traverse.c
+++ b/lib/tdb/test/run-nested-traverse.c
@@ -41,7 +41,30 @@ static int traverse2(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
return 0;
}
-static int traverse1(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+static int traverse1r(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+ void *p)
+{
+ ok1(correct_key(key));
+ ok1(correct_data(data));
+ ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
+ == SUCCESS);
+ ok1(external_agent_operation(agent, STORE, tdb_name(tdb))
+ == SUCCESS);
+ ok1(external_agent_operation(agent, TRANSACTION_COMMIT, tdb_name(tdb))
+ == WOULD_HAVE_BLOCKED);
+ tdb_traverse(tdb, traverse2, NULL);
+
+ /* That should *not* release the all-records lock! */
+ ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
+ == SUCCESS);
+ ok1(external_agent_operation(agent, STORE, tdb_name(tdb))
+ == SUCCESS);
+ ok1(external_agent_operation(agent, TRANSACTION_COMMIT, tdb_name(tdb))
+ == WOULD_HAVE_BLOCKED);
+ return 0;
+}
+
+static int traverse1w(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
void *p)
{
ok1(correct_key(key));
@@ -50,7 +73,7 @@ static int traverse1(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
== WOULD_HAVE_BLOCKED);
tdb_traverse(tdb, traverse2, NULL);
- /* That should *not* release the transaction lock! */
+ /* That should *not* release the all-records lock! */
ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
== WOULD_HAVE_BLOCKED);
return 0;
@@ -80,8 +103,8 @@ int main(int argc, char *argv[])
data.dsize = strlen("world");
ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
- tdb_traverse(tdb, traverse1, NULL);
- tdb_traverse_read(tdb, traverse1, NULL);
+ tdb_traverse(tdb, traverse1w, NULL);
+ tdb_traverse_read(tdb, traverse1r, NULL);
tdb_close(tdb);
return exit_status();
--
1.9.1
From cd0d66680d5b24004f06f45bdab09ea63f94fca2 Mon Sep 17 00:00:00 2001
From: Stefan Metzmacher <metze at samba.org>
Date: Tue, 11 Apr 2017 17:27:33 +0200
Subject: [PATCH 06/17] tdb: version 1.3.13
* documentation for the tdbbackup -n option
* correctly upgrade F_RDLCK to F_WRLCK locks
* allow tdb_traverse_read before tdb_transaction[_prepare]_commit()
Signed-off-by: Stefan Metzmacher <metze at samba.org>
---
lib/tdb/ABI/tdb-1.3.13.sigs | 70 +++++++++++++++++++++++++++++++++++++++++++++
lib/tdb/wscript | 2 +-
2 files changed, 71 insertions(+), 1 deletion(-)
create mode 100644 lib/tdb/ABI/tdb-1.3.13.sigs
diff --git a/lib/tdb/ABI/tdb-1.3.13.sigs b/lib/tdb/ABI/tdb-1.3.13.sigs
new file mode 100644
index 0000000..48f4278
--- /dev/null
+++ b/lib/tdb/ABI/tdb-1.3.13.sigs
@@ -0,0 +1,70 @@
+tdb_add_flags: void (struct tdb_context *, unsigned int)
+tdb_append: int (struct tdb_context *, TDB_DATA, TDB_DATA)
+tdb_chainlock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_mark: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_nonblock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_read: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_read_nonblock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_unmark: int (struct tdb_context *, TDB_DATA)
+tdb_chainunlock: int (struct tdb_context *, TDB_DATA)
+tdb_chainunlock_read: int (struct tdb_context *, TDB_DATA)
+tdb_check: int (struct tdb_context *, int (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_close: int (struct tdb_context *)
+tdb_delete: int (struct tdb_context *, TDB_DATA)
+tdb_dump_all: void (struct tdb_context *)
+tdb_enable_seqnum: void (struct tdb_context *)
+tdb_error: enum TDB_ERROR (struct tdb_context *)
+tdb_errorstr: const char *(struct tdb_context *)
+tdb_exists: int (struct tdb_context *, TDB_DATA)
+tdb_fd: int (struct tdb_context *)
+tdb_fetch: TDB_DATA (struct tdb_context *, TDB_DATA)
+tdb_firstkey: TDB_DATA (struct tdb_context *)
+tdb_freelist_size: int (struct tdb_context *)
+tdb_get_flags: int (struct tdb_context *)
+tdb_get_logging_private: void *(struct tdb_context *)
+tdb_get_seqnum: int (struct tdb_context *)
+tdb_hash_size: int (struct tdb_context *)
+tdb_increment_seqnum_nonblock: void (struct tdb_context *)
+tdb_jenkins_hash: unsigned int (TDB_DATA *)
+tdb_lock_nonblock: int (struct tdb_context *, int, int)
+tdb_lockall: int (struct tdb_context *)
+tdb_lockall_mark: int (struct tdb_context *)
+tdb_lockall_nonblock: int (struct tdb_context *)
+tdb_lockall_read: int (struct tdb_context *)
+tdb_lockall_read_nonblock: int (struct tdb_context *)
+tdb_lockall_unmark: int (struct tdb_context *)
+tdb_log_fn: tdb_log_func (struct tdb_context *)
+tdb_map_size: size_t (struct tdb_context *)
+tdb_name: const char *(struct tdb_context *)
+tdb_nextkey: TDB_DATA (struct tdb_context *, TDB_DATA)
+tdb_null: dptr = 0xXXXX, dsize = 0
+tdb_open: struct tdb_context *(const char *, int, int, int, mode_t)
+tdb_open_ex: struct tdb_context *(const char *, int, int, int, mode_t, const struct tdb_logging_context *, tdb_hash_func)
+tdb_parse_record: int (struct tdb_context *, TDB_DATA, int (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_printfreelist: int (struct tdb_context *)
+tdb_remove_flags: void (struct tdb_context *, unsigned int)
+tdb_reopen: int (struct tdb_context *)
+tdb_reopen_all: int (int)
+tdb_repack: int (struct tdb_context *)
+tdb_rescue: int (struct tdb_context *, void (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_runtime_check_for_robust_mutexes: bool (void)
+tdb_set_logging_function: void (struct tdb_context *, const struct tdb_logging_context *)
+tdb_set_max_dead: void (struct tdb_context *, int)
+tdb_setalarm_sigptr: void (struct tdb_context *, volatile sig_atomic_t *)
+tdb_store: int (struct tdb_context *, TDB_DATA, TDB_DATA, int)
+tdb_storev: int (struct tdb_context *, TDB_DATA, const TDB_DATA *, int, int)
+tdb_summary: char *(struct tdb_context *)
+tdb_transaction_cancel: int (struct tdb_context *)
+tdb_transaction_commit: int (struct tdb_context *)
+tdb_transaction_prepare_commit: int (struct tdb_context *)
+tdb_transaction_start: int (struct tdb_context *)
+tdb_transaction_start_nonblock: int (struct tdb_context *)
+tdb_transaction_write_lock_mark: int (struct tdb_context *)
+tdb_transaction_write_lock_unmark: int (struct tdb_context *)
+tdb_traverse: int (struct tdb_context *, tdb_traverse_func, void *)
+tdb_traverse_read: int (struct tdb_context *, tdb_traverse_func, void *)
+tdb_unlock: int (struct tdb_context *, int, int)
+tdb_unlockall: int (struct tdb_context *)
+tdb_unlockall_read: int (struct tdb_context *)
+tdb_validate_freelist: int (struct tdb_context *, int *)
+tdb_wipe_all: int (struct tdb_context *)
diff --git a/lib/tdb/wscript b/lib/tdb/wscript
index 79b0e6f..09bc0a3 100644
--- a/lib/tdb/wscript
+++ b/lib/tdb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'tdb'
-VERSION = '1.3.12'
+VERSION = '1.3.13'
blddir = 'bin'
--
1.9.1
From b8cd978a7707599f215727b71880a9a0849ed752 Mon Sep 17 00:00:00 2001
From: Garming Sam <garming at catalyst.net.nz>
Date: Thu, 30 Mar 2017 12:03:17 +1300
Subject: [PATCH 07/17] ldb_tdb: Ensure we correctly decrement
ltdb->read_lock_count
If we do not do this, then we never take the all record lock, and instead do a lock
for every record as we go, which is very slow during a large search
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
Signed-off-by: Garming Sam <garming at catalyst.net.nz>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 6b1187e..7febf07 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -119,6 +119,7 @@ int ltdb_unlock_read(struct ldb_module *module)
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
tdb_unlockall_read(ltdb->tdb);
+ ltdb->read_lock_count--;
return 0;
}
ltdb->read_lock_count--;
--
1.9.1
From 70e4c27f48318f1fae76061ee391ba2ff471496c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 14:26:23 +1300
Subject: [PATCH 08/17] ldb_tdb: Provide better debugging on prepare_commit
failures
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 7febf07..8b2f099 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1124,6 +1124,7 @@ static int ltdb_start_trans(struct ldb_module *module)
static int ltdb_prepare_commit(struct ldb_module *module)
{
+ int ret;
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
@@ -1131,15 +1132,21 @@ static int ltdb_prepare_commit(struct ldb_module *module)
return LDB_SUCCESS;
}
- if (ltdb_index_transaction_commit(module) != 0) {
+ ret = ltdb_index_transaction_commit(module);
+ if (ret != LDB_SUCCESS) {
tdb_transaction_cancel(ltdb->tdb);
ltdb->in_transaction--;
- return ltdb_err_map(tdb_error(ltdb->tdb));
+ return ret;
}
if (tdb_transaction_prepare_commit(ltdb->tdb) != 0) {
+ ret = ltdb_err_map(tdb_error(ltdb->tdb));
ltdb->in_transaction--;
- return ltdb_err_map(tdb_error(ltdb->tdb));
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Failure during tdb_transaction_prepare_commit(): %s -> %s",
+ tdb_errorstr(ltdb->tdb),
+ ldb_strerror(ret));
+ return ret;
}
ltdb->prepared_commit = true;
--
1.9.1
From 4ede4a4aa0a676b94e5c7c0d885497cb02ed627d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 14:27:55 +1300
Subject: [PATCH 09/17] ldb_tdb: Provide better debugging on end_trans failures
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_tdb.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 8b2f099..58109d0 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1156,11 +1156,12 @@ static int ltdb_prepare_commit(struct ldb_module *module)
static int ltdb_end_trans(struct ldb_module *module)
{
+ int ret;
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
if (!ltdb->prepared_commit) {
- int ret = ltdb_prepare_commit(module);
+ ret = ltdb_prepare_commit(module);
if (ret != LDB_SUCCESS) {
return ret;
}
@@ -1170,7 +1171,12 @@ static int ltdb_end_trans(struct ldb_module *module)
ltdb->prepared_commit = false;
if (tdb_transaction_commit(ltdb->tdb) != 0) {
- return ltdb_err_map(tdb_error(ltdb->tdb));
+ ret = ltdb_err_map(tdb_error(ltdb->tdb));
+ ldb_asprintf_errstring(ldb_module_get_ctx(module),
+ "Failure during tdb_transaction_commit(): %s -> %s",
+ tdb_errorstr(ltdb->tdb),
+ ldb_strerror(ret));
+ return ret;
}
return LDB_SUCCESS;
--
1.9.1
From cac0ba709e432444a2e77810accc1f789ad64b4b Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:10:08 +1300
Subject: [PATCH 10/17] ldb_tdb: Split index load out into a sub-funciton:
ltdb_index_load
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 73 +++++++++++++++++++++++++++------------------
1 file changed, 44 insertions(+), 29 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 5ea09d6..8928cca 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -226,6 +226,49 @@ failed:
return -1;
}
+/*
+ register any index records we find for the DB
+*/
+static int ltdb_index_load(struct ldb_module *module,
+ struct ltdb_private *ltdb)
+{
+ struct ldb_dn *indexlist_dn;
+ int r;
+
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+
+ talloc_free(ltdb->cache->indexlist);
+
+ ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
+ if (ltdb->cache->indexlist == NULL) {
+ return -1;
+ }
+ ltdb->cache->one_level_indexes = false;
+ ltdb->cache->attribute_indexes = false;
+
+ indexlist_dn = ldb_dn_new(ltdb, ldb, LTDB_INDEXLIST);
+ if (indexlist_dn == NULL) {
+ return -1;
+ }
+
+ r = ltdb_search_dn1(module, indexlist_dn, ltdb->cache->indexlist,
+ LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
+ |LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC
+ |LDB_UNPACK_DATA_FLAG_NO_DN);
+ TALLOC_FREE(indexlist_dn);
+
+ if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
+ return -1;
+ }
+
+ if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXONE) != NULL) {
+ ltdb->cache->one_level_indexes = true;
+ }
+ if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
+ ltdb->cache->attribute_indexes = true;
+ }
+ return 0;
+}
/*
initialise the baseinfo record
@@ -316,7 +359,6 @@ int ltdb_cache_load(struct ldb_module *module)
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
struct ldb_dn *baseinfo_dn = NULL, *options_dn = NULL;
- struct ldb_dn *indexlist_dn = NULL;
uint64_t seq;
struct ldb_message *baseinfo = NULL, *options = NULL;
int r;
@@ -332,10 +374,6 @@ int ltdb_cache_load(struct ldb_module *module)
if (ltdb->cache == NULL) {
ltdb->cache = talloc_zero(ltdb, struct ltdb_cache);
if (ltdb->cache == NULL) goto failed;
- ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
- if (ltdb->cache->indexlist == NULL) {
- goto failed;
- }
}
baseinfo = ldb_msg_new(ltdb->cache);
@@ -402,7 +440,6 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb->disallow_dn_filter = false;
}
- talloc_free(ltdb->cache->indexlist);
/*
* ltdb_attributes_unload() calls internally talloc_free() on
* any non-fixed elemnts in ldb->schema.attributes.
@@ -412,31 +449,11 @@ int ltdb_cache_load(struct ldb_module *module)
* partition module.
*/
ltdb_attributes_unload(module);
- ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
- if (ltdb->cache->indexlist == NULL) {
- goto failed;
- }
- ltdb->cache->one_level_indexes = false;
- ltdb->cache->attribute_indexes = false;
-
- indexlist_dn = ldb_dn_new(module, ldb, LTDB_INDEXLIST);
- if (indexlist_dn == NULL) goto failed;
- r = ltdb_search_dn1(module, indexlist_dn, ltdb->cache->indexlist,
- LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
- |LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC
- |LDB_UNPACK_DATA_FLAG_NO_DN);
- if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
+ if (ltdb_index_load(module, ltdb) == -1) {
goto failed;
}
- if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXONE) != NULL) {
- ltdb->cache->one_level_indexes = true;
- }
- if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
- ltdb->cache->attribute_indexes = true;
- }
-
/*
* NOTE WELL: This is per-ldb, not per module, so overwrites
* the handlers across all databases when used under Samba's
@@ -449,13 +466,11 @@ int ltdb_cache_load(struct ldb_module *module)
done:
talloc_free(options);
talloc_free(baseinfo);
- talloc_free(indexlist_dn);
return 0;
failed:
talloc_free(options);
talloc_free(baseinfo);
- talloc_free(indexlist_dn);
return -1;
}
--
1.9.1
From 24e45bf87dbd6eaad87085dffcc693f44adca678 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:07:16 +1300
Subject: [PATCH 11/17] ldb_tdb: change the arguments to ldb_is_indexed() to
provide the ltdb_private
By doing this, we can be more efficient in locating if we have an index in
the future.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 58 ++++++++++++++++++++++++++-------------------
1 file changed, 33 insertions(+), 25 deletions(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 53fcde5..8912aea 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -445,12 +445,16 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
/*
see if a attribute value is in the list of indexed attributes
*/
-static bool ltdb_is_indexed(const struct ldb_message *index_list, const char *attr)
+static bool ltdb_is_indexed(struct ldb_module *module,
+ struct ltdb_private *ltdb,
+ const char *attr)
{
unsigned int i;
struct ldb_message_element *el;
+ struct ldb_context *ldb = ldb_module_get_ctx(module);
+
- el = ldb_msg_find_element(index_list, LTDB_IDXATTR);
+ el = ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR);
if (el == NULL) {
return false;
}
@@ -481,8 +485,8 @@ static bool ltdb_is_indexed(const struct ldb_message *index_list, const char *at
equality search only)
*/
static int ltdb_index_dn_simple(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
struct ldb_context *ldb;
@@ -496,7 +500,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
/* if the attribute isn't in the list of indexed attributes then
this node needs a full search */
- if (!ltdb_is_indexed(index_list, tree->u.equality.attr)) {
+ if (!ltdb_is_indexed(module, ltdb, tree->u.equality.attr)) {
return LDB_ERR_OPERATIONS_ERROR;
}
@@ -517,12 +521,10 @@ static bool list_union(struct ldb_context *, struct dn_list *, const struct dn_l
return a list of dn's that might match a leaf indexed search
*/
static int ltdb_index_dn_leaf(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
- struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
- struct ltdb_private);
if (ltdb->disallow_dn_filter &&
(ldb_attr_cmp(tree->u.equality.attr, "dn") == 0)) {
/* in AD mode we do not support "(dn=...)" search filters */
@@ -540,7 +542,7 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
list->count = 1;
return LDB_SUCCESS;
}
- return ltdb_index_dn_simple(module, tree, index_list, list);
+ return ltdb_index_dn_simple(module, ltdb, tree, list);
}
@@ -653,8 +655,8 @@ static bool list_union(struct ldb_context *ldb,
}
static int ltdb_index_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list);
@@ -662,8 +664,8 @@ static int ltdb_index_dn(struct ldb_module *module,
process an OR list (a union)
*/
static int ltdb_index_dn_or(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
struct ldb_context *ldb;
@@ -683,7 +685,8 @@ static int ltdb_index_dn_or(struct ldb_module *module,
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_dn(module, tree->u.list.elements[i], index_list, list2);
+ ret = ltdb_index_dn(module, ltdb,
+ tree->u.list.elements[i], list2);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* X || 0 == X */
@@ -715,8 +718,8 @@ static int ltdb_index_dn_or(struct ldb_module *module,
NOT an index results
*/
static int ltdb_index_dn_not(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
/* the only way to do an indexed not would be if we could
@@ -746,8 +749,8 @@ static bool ltdb_index_unique(struct ldb_context *ldb,
process an AND expression (intersection)
*/
static int ltdb_index_dn_and(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
struct ldb_context *ldb;
@@ -771,7 +774,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
continue;
}
- ret = ltdb_index_dn(module, subtree, index_list, list);
+ ret = ltdb_index_dn(module, ltdb,
+ subtree, list);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* 0 && X == 0 */
return LDB_ERR_NO_SUCH_OBJECT;
@@ -798,7 +802,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
return ldb_module_oom(module);
}
- ret = ltdb_index_dn(module, subtree, index_list, list2);
+ ret = ltdb_index_dn(module, ltdb,
+ subtree, list2);
if (ret == LDB_ERR_NO_SUCH_OBJECT) {
/* X && 0 == 0 */
@@ -884,27 +889,27 @@ static int ltdb_index_dn_one(struct ldb_module *module,
an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
*/
static int ltdb_index_dn(struct ldb_module *module,
+ struct ltdb_private *ltdb,
const struct ldb_parse_tree *tree,
- const struct ldb_message *index_list,
struct dn_list *list)
{
int ret = LDB_ERR_OPERATIONS_ERROR;
switch (tree->operation) {
case LDB_OP_AND:
- ret = ltdb_index_dn_and(module, tree, index_list, list);
+ ret = ltdb_index_dn_and(module, ltdb, tree, list);
break;
case LDB_OP_OR:
- ret = ltdb_index_dn_or(module, tree, index_list, list);
+ ret = ltdb_index_dn_or(module, ltdb, tree, list);
break;
case LDB_OP_NOT:
- ret = ltdb_index_dn_not(module, tree, index_list, list);
+ ret = ltdb_index_dn_not(module, ltdb, tree, list);
break;
case LDB_OP_EQUALITY:
- ret = ltdb_index_dn_leaf(module, tree, index_list, list);
+ ret = ltdb_index_dn_leaf(module, ltdb, tree, list);
break;
case LDB_OP_SUBSTRING:
@@ -1087,7 +1092,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
talloc_free(dn_list);
return LDB_ERR_OPERATIONS_ERROR;
}
- ret = ltdb_index_dn(ac->module, ac->tree, ltdb->cache->indexlist, dn_list);
+ ret = ltdb_index_dn(ac->module, ltdb, ac->tree, dn_list);
if (ret != LDB_SUCCESS) {
talloc_free(dn_list);
return ret;
@@ -1221,14 +1226,15 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
return LDB_SUCCESS;
}
- if (ltdb->cache->indexlist->num_elements == 0) {
+ if (ltdb->cache->attribute_indexes == false) {
/* no indexed fields */
return LDB_SUCCESS;
}
for (i = 0; i < num_el; i++) {
int ret;
- if (!ltdb_is_indexed(ltdb->cache->indexlist, elements[i].name)) {
+ if (!ltdb_is_indexed(module,
+ ltdb, elements[i].name)) {
continue;
}
ret = ltdb_index_add_el(module, dn, &elements[i], is_new);
@@ -1306,7 +1312,8 @@ int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
if (ldb_dn_is_special(dn)) {
return LDB_SUCCESS;
}
- if (!ltdb_is_indexed(ltdb->cache->indexlist, el->name)) {
+ if (!ltdb_is_indexed(module,
+ ltdb, el->name)) {
return LDB_SUCCESS;
}
return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el, true);
@@ -1439,7 +1446,8 @@ int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
return LDB_SUCCESS;
}
- if (!ltdb_is_indexed(ltdb->cache->indexlist, el->name)) {
+ if (!ltdb_is_indexed(module,
+ ltdb, el->name)) {
return LDB_SUCCESS;
}
for (i = 0; i < el->num_values; i++) {
--
1.9.1
From 3df355cba66e72b8850ed8d07f9c03f60ca82123 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:21:34 +1300
Subject: [PATCH 12/17] ldb_tdb: consistently use
ltdb->cache->attribute_indexes to determine if we have indexes
This is instead of checking the number of elements via ltdb->cache->indexlist->num_elements
In turn, this allows us to avoid fetching ltdb->cache->indexlist in the future
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_index.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8912aea..e121b8a 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -453,6 +453,9 @@ static bool ltdb_is_indexed(struct ldb_module *module,
struct ldb_message_element *el;
struct ldb_context *ldb = ldb_module_get_ctx(module);
+ if (ltdb->cache->attribute_indexes == false) {
+ return false;
+ }
el = ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR);
if (el == NULL) {
@@ -1647,7 +1650,7 @@ int ltdb_reindex(struct ldb_module *module)
}
/* if we don't have indexes we have nothing todo */
- if (ltdb->cache->indexlist->num_elements == 0) {
+ if (ltdb->cache->attribute_indexes == false) {
return LDB_SUCCESS;
}
--
1.9.1
From c229fbc008c39ce56467ce302b2d5a17d123663e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:23:44 +1300
Subject: [PATCH 13/17] ldb: Allow a caller (in particular Samba) to handle the
list of attributes with an index
By doing that, Samba will use a binary search to locate the attributes
rather than an O(n) search, during every search or modify of the database.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/common/ldb_attributes.c | 12 ++++++++++++
lib/ldb/include/ldb.h | 5 +++++
lib/ldb/include/ldb_module.h | 2 ++
lib/ldb/include/ldb_private.h | 7 +++++++
lib/ldb/ldb_tdb/ldb_index.c | 9 +++++++++
5 files changed, 35 insertions(+)
diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c
index 2021a0b..98ec5a4 100644
--- a/lib/ldb/common/ldb_attributes.c
+++ b/lib/ldb/common/ldb_attributes.c
@@ -383,3 +383,15 @@ void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
ldb->schema.attribute_handler_override_private = private_data;
ldb->schema.attribute_handler_override = override;
}
+
+/*
+ set that the attribute handler override function - used to delegate
+ schema handling to external code, is handling setting
+ LDB_ATTR_FLAG_INDEXED
+ */
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+ bool one_level_indexes)
+{
+ ldb->schema.index_handler_override = true;
+ ldb->schema.one_level_indexes = one_level_indexes;
+}
diff --git a/lib/ldb/include/ldb.h b/lib/ldb/include/ldb.h
index afcf9eb..a3babca 100644
--- a/lib/ldb/include/ldb.h
+++ b/lib/ldb/include/ldb.h
@@ -441,6 +441,11 @@ const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_c
*/
#define LDB_ATTR_FLAG_FROM_DB (1<<6)
+/*
+ * The attribute was loaded from a DB, rather than via the C API
+ */
+#define LDB_ATTR_FLAG_INDEXED (1<<7)
+
/**
LDAP attribute syntax for a DN
diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index 833d5a8..75f3fcb 100644
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -125,6 +125,8 @@ typedef const struct ldb_schema_attribute *(*ldb_attribute_handler_override_fn_t
void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
ldb_attribute_handler_override_fn_t override,
void *private_data);
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+ bool one_level_indexes);
/* A useful function to build comparison functions with */
int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx,
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index 644d49c..bd975b8 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -88,6 +88,13 @@ struct ldb_schema {
unsigned num_dn_extended_syntax;
struct ldb_dn_extended_syntax *dn_extended_syntax;
+
+ /*
+ * If set, the attribute_handler_override has the details of
+ * what attributes have an index
+ */
+ bool index_handler_override;
+ bool one_level_indexes;
};
/*
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index e121b8a..dd7c2db 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -453,6 +453,15 @@ static bool ltdb_is_indexed(struct ldb_module *module,
struct ldb_message_element *el;
struct ldb_context *ldb = ldb_module_get_ctx(module);
+ if (ldb->schema.index_handler_override) {
+ const struct ldb_schema_attribute *a
+ = ldb_schema_attribute_by_name(ldb, attr);
+ if (a->flags & LDB_ATTR_FLAG_INDEXED) {
+ return true;
+ } else {
+ return false;
+ }
+ }
if (ltdb->cache->attribute_indexes == false) {
return false;
}
--
1.9.1
From 44eac88f5f4faec2af54fe333207e133dfe9ad54 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:10:22 +1300
Subject: [PATCH 14/17] ldb_tdb: Avoid reading the index list from the DB if we
are already set to override it
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 8928cca..7b5e1fc 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -237,6 +237,14 @@ static int ltdb_index_load(struct ldb_module *module,
struct ldb_context *ldb = ldb_module_get_ctx(module);
+ if (ldb->schema.index_handler_override) {
+ ltdb->cache->attribute_indexes = true;
+ ltdb->cache->one_level_indexes = ldb->schema.one_level_indexes;
+ /* we skip loading the @INDEXLIST record when a module is supplying
+ its own attribute handling */
+ return 0;
+ }
+
talloc_free(ltdb->cache->indexlist);
ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
--
1.9.1
From 1db664eb84351c548d4d65bbbfff8e30ae8ded01 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:11:15 +1300
Subject: [PATCH 15/17] ldb_tdb: Call talloc_free(options_dn) as soon as we are
done with options_dn
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ldb_tdb/ldb_cache.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 7b5e1fc..dfef37e 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -431,6 +431,7 @@ int ltdb_cache_load(struct ldb_module *module)
if (options_dn == NULL) goto failed;
r= ltdb_search_dn1(module, options_dn, options, 0);
+ talloc_free(options_dn);
if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
goto failed;
}
--
1.9.1
From 44881fd8ad7ba84ac47715db4e8e8ea3a02cf17a Mon Sep 17 00:00:00 2001
From: Stefan Metzmacher <metze at samba.org>
Date: Tue, 11 Apr 2017 17:50:08 +0200
Subject: [PATCH 16/17] ldb: version 1.1.30
* let ldbdump parse the -i option
* don't allow the reveal_internals control for ldbedit
* only allow --show-binary for ldbsearch
* don't let ldbsearch create non-existing files
* fix ldb_tdb locking (performance) problems
* providee an interface for improved indexing for callers
like Samba, which will allow much better performance.
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
lib/ldb/ABI/ldb-1.1.30.sigs | 269 +++++++++++++++++++++++++++++++++
lib/ldb/ABI/pyldb-util-1.1.30.sigs | 2 +
lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs | 2 +
lib/ldb/wscript | 2 +-
4 files changed, 274 insertions(+), 1 deletion(-)
create mode 100644 lib/ldb/ABI/ldb-1.1.30.sigs
create mode 100644 lib/ldb/ABI/pyldb-util-1.1.30.sigs
create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs
diff --git a/lib/ldb/ABI/ldb-1.1.30.sigs b/lib/ldb/ABI/ldb-1.1.30.sigs
new file mode 100644
index 0000000..9b865ed8
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.1.30.sigs
@@ -0,0 +1,269 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.1.30.sigs b/lib/ldb/ABI/pyldb-util-1.1.30.sigs
new file mode 100644
index 0000000..74d6719
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.1.30.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs b/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs
new file mode 100644
index 0000000..74d6719
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index c877f3b..479e46d 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
#!/usr/bin/env python
APPNAME = 'ldb'
-VERSION = '1.1.29'
+VERSION = '1.1.30'
blddir = 'bin'
--
1.9.1
From 3185e6dc2ba488897dc03d19865ff618ea430d9a Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet at samba.org>
Date: Thu, 30 Mar 2017 13:25:35 +1300
Subject: [PATCH 17/17] schema: Use ldb_schema_set_override_indexlist for
faster index selection
This allows Samba to provide a binary tree lookup for the existance of an index on the attribute
rather than the O(n) lookup that was being done for each attribute during a search or modify
Signed-off-by: Andrew Bartlett <abartlet at samba.org>
---
source4/dsdb/schema/schema_init.c | 4 ++++
source4/dsdb/schema/schema_set.c | 1 +
2 files changed, 5 insertions(+)
diff --git a/source4/dsdb/schema/schema_init.c b/source4/dsdb/schema/schema_init.c
index 256c467..c76b57c 100644
--- a/source4/dsdb/schema/schema_init.c
+++ b/source4/dsdb/schema/schema_init.c
@@ -514,6 +514,10 @@ static int dsdb_schema_setup_ldb_schema_attribute(struct ldb_context *ldb,
a->flags |= LDB_ATTR_FLAG_SINGLE_VALUE;
}
+ if (attr->searchFlags & SEARCH_FLAG_ATTINDEX) {
+ a->flags |= LDB_ATTR_FLAG_INDEXED;
+ }
+
return LDB_SUCCESS;
}
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 2e688d0..f43ec5f 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -69,6 +69,7 @@ static int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb, struc
/* setup our own attribute name to schema handler */
ldb_schema_attribute_set_override_handler(ldb, dsdb_attribute_handler_override, schema);
+ ldb_schema_set_override_indexlist(ldb, true);
if (!write_indices_and_attributes) {
return ret;
--
1.9.1
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 836 bytes
Desc: OpenPGP digital signature
URL: <http://lists.samba.org/pipermail/samba-technical/attachments/20170411/508b8871/signature.sig>
More information about the samba-technical
mailing list