[SCM] CTDB repository - branch 1.2.40 updated - ctdb-1.2.55-2-g0e99ca1

[Amitay Isaacs]

The branch, 1.2.40 has been updated
       via  0e99ca1cdf28a7043554afb78bd439f727ab4f95 (commit)
       via  e80b2c15bf8c8fb5c3793acfebbe09d3cdd617b7 (commit)
      from  8b2d84482bacd3b31db013496ce82c2e7b730e86 (commit)

http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=1.2.40


- Log -----------------------------------------------------------------
commit 0e99ca1cdf28a7043554afb78bd439f727ab4f95
Author: Amitay Isaacs <amitay at gmail.com>
Date:   Wed Dec 5 11:38:42 2012 +1100

    scripts: Add helper script to log locking information using /proc/locks
    
    This finds any processes locking tdb databases used by CTDB and logs
    stack trace for each process.
    
    Signed-off-by: Amitay Isaacs <amitay at gmail.com>

commit e80b2c15bf8c8fb5c3793acfebbe09d3cdd617b7
Author: Amitay Isaacs <amitay at gmail.com>
Date:   Wed Dec 5 11:37:26 2012 +1100

    daemon: Run an external script if freeze locks were not obtained during recovery
    
    If the freeze child is already created in ctdb_start_freeze(), then it indicates
    that the child process has not yet obtained the locks.  This may be because
    another process has locked the databases and has not yet released the locks.
    
    In this case, invoke a helper script defined by environmental variable
    CTDB_DEBUG_LOCKS, to log information about locks.
    
    Signed-off-by: Amitay Isaacs <amitay at gmail.com>

-----------------------------------------------------------------------

Summary of changes:
 Makefile.in                |    1 +
 config/debug_locks.sh      |   38 ++++++++++++++++++++++++++++++++++++++
 packaging/RPM/ctdb.spec.in |    1 +
 server/ctdb_freeze.c       |   24 ++++++++++++++++++++++++
 4 files changed, 64 insertions(+), 0 deletions(-)
 create mode 100755 config/debug_locks.sh


Changeset truncated at 500 lines:

diff --git a/Makefile.in b/Makefile.in
index e3d6d5b..136ae30 100755
--- a/Makefile.in
+++ b/Makefile.in
@@ -264,6 +264,7 @@ install: all
 	${INSTALLCMD} -m 644 config/functions $(DESTDIR)$(etcdir)/ctdb
 	${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb
 	${INSTALLCMD} -m 755 config/interface_modify.sh $(DESTDIR)$(etcdir)/ctdb
+	${INSTALLCMD} -m 755 config/debug_locks.sh $(DESTDIR)$(etcdir)/ctdb
 	${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts
 	${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt
 	${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d
diff --git a/config/debug_locks.sh b/config/debug_locks.sh
new file mode 100755
index 0000000..5b9efed
--- /dev/null
+++ b/config/debug_locks.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Create sed expression to convert inodes to names
+sed_cmd=$( ls -li /var/ctdb/*.tdb.* /var/ctdb/persistent/*.tdb.* |
+	   sed -e "s#/var/ctdb[/persistent]*/\(.*\)#\1#" |
+	   awk '{printf "s#[0-9]*:[0-9]*:%s #%s #\n", $1, $10}' )
+
+# Parse /proc/locks and extract following information
+#    pid process_name tdb_name offsets [W]
+out=$( cat /proc/locks |
+    grep -F "POSIX  ADVISORY  WRITE" |
+    awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' |
+    while read pid rest ; do
+	pname=$(readlink /proc/$pid/exe)
+	echo $pid $pname $rest
+    done | sed -e "$sed_fu" | grep "\.tdb" )
+
+if [ -n "$out" ]; then
+    # Log information about locks
+    echo "$out" | logger -t "debug-lock"
+
+    # Find processes that are waiting for locks
+    dbs=$(echo "$out" | grep "W$" | awk '{print $3}')
+    all_pids=""
+    for db in $dbs ; do
+	pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}')
+	all_pids="$all_pids $pids"
+    done
+    pids=$(echo $all_pids | sort -u)
+
+    # For each process waiting, log stack trace
+    for pid in $pids ; do
+	gstack $pid | logger -t "debug-lock $pid"
+#	gcore -o /var/log/core-deadlock-ctdb $pid
+    done
+fi
+
+exit 0
diff --git a/packaging/RPM/ctdb.spec.in b/packaging/RPM/ctdb.spec.in
index de8193d..39a5df9 100644
--- a/packaging/RPM/ctdb.spec.in
+++ b/packaging/RPM/ctdb.spec.in
@@ -113,6 +113,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_sysconfdir}/ctdb/events.d/91.lvs
 %{_sysconfdir}/ctdb/statd-callout
 %{_sysconfdir}/ctdb/interface_modify.sh
+%{_sysconfdir}/ctdb/debug_locks.sh
 %{_sbindir}/ctdbd
 %{_bindir}/ctdb
 %{_bindir}/smnotify
diff --git a/server/ctdb_freeze.c b/server/ctdb_freeze.c
index 0f70fd3..f422e6d 100644
--- a/server/ctdb_freeze.c
+++ b/server/ctdb_freeze.c
@@ -256,6 +256,26 @@ static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 }
 
 /*
+ * Run an external script to check if there is a deadlock situation
+ */
+static void ctdb_debug_locks(void)
+{
+	const char *cmd = getenv("CTDB_DEBUG_LOCKS");
+	int pid;
+
+	if (cmd == NULL) {
+		return;
+	}
+
+	pid = fork();
+
+	/* Execute only in child process */
+	if (pid == 0) {
+		execl(cmd, cmd, NULL);
+	}
+}
+
+/*
   start the freeze process for a certain priority
  */
 int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
@@ -283,6 +303,10 @@ int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 		ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
 		CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
 		ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
+	} else {
+		/* The previous free lock child has not yet been able to get locks.
+		 * Invoke debugging script */
+		ctdb_debug_locks();
 	}
 
 	return 0;


-- 
CTDB repository