[SCM] CTDB repository - branch master updated - ctdb-1.10-72-g9463e04

Ronnie Sahlberg sahlberg at samba.org
Fri Jan 14 00:14:23 MST 2011


The branch, master has been updated
       via  9463e04038ba36792583f83bd95c1af322dc283a (commit)
       via  2c02fc2d45cd7364d7bee0d6a89f1386131ef002 (commit)
       via  70154e5e19e219de086b2995d41e8f6e069ee20d (commit)
       via  6954c9df67501183995f408cca358c8fdfb176ab (commit)
      from  d8af74e4c4961deb94c18dde8ba7fc07e944729c (commit)

http://gitweb.samba.org/?p=sahlberg/ctdb.git;a=shortlog;h=master


- Log -----------------------------------------------------------------
commit 9463e04038ba36792583f83bd95c1af322dc283a
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date:   Fri Jan 14 17:35:31 2011 +1100

    LIBCTDB: add support for traverse

commit 2c02fc2d45cd7364d7bee0d6a89f1386131ef002
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date:   Fri Jan 14 09:53:25 2011 +1100

    db_exists() takes 3 arguments, not two.

commit 70154e5e19e219de086b2995d41e8f6e069ee20d
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date:   Fri Jan 14 09:46:04 2011 +1100

    We can not always rely on the recovery daemon pinging us in a timely manner
    so we need a "ticker" in the main ctdbd daemon too to ensure we get at least one event to process every second.
    
    This will improve the accuracy of "Time jumped" messages and remove false positives when the recovery daemon is "slow".

commit 6954c9df67501183995f408cca358c8fdfb176ab
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date:   Thu Jan 13 16:17:43 2011 +1100

    ADDIP failure
    
    Found during automatic regression testing.
    We do not allow the takeip/releaseip events to be executed during a recovery.
    
    All of "ctdb addip, ctdb delip, ctdb moveip" use and force these events to
    trigger to perform the ip assignments required.
    
    If these commands collide with a recovery, these commands could fail since we do
    not allow takeip/releaseip events to trigger during the recovery.
    While it is easy to just try running hte command again, this is suboptimal for script use.
    
    Change these commands to retry these operations a few times until either successfull or until we give up.
    This makes the commands much easier to use in scripts.

-----------------------------------------------------------------------

Summary of changes:
 include/ctdb.h          |   42 +++++++++++
 include/ctdb_private.h  |   18 -----
 include/ctdb_protocol.h |   20 +++++
 libctdb/ctdb.c          |  183 +++++++++++++++++++++++++++++++++++++++++++++++
 libctdb/tst.c           |   22 ++++++
 server/ctdb_daemon.c    |   30 ++++++++
 tools/ctdb.c            |   76 +++++++++++++++----
 7 files changed, 357 insertions(+), 34 deletions(-)


Changeset truncated at 500 lines:

diff --git a/include/ctdb.h b/include/ctdb.h
index 1c6cf35..c95c2e1 100644
--- a/include/ctdb.h
+++ b/include/ctdb.h
@@ -291,6 +291,47 @@ bool ctdb_writerecord(struct ctdb_db *ctdb_db,
  */
 void ctdb_release_lock(struct ctdb_db *ctdb_db, struct ctdb_lock *lock);
 
+
+
+/**
+ * ctdb_traverse_callback_t - callback for ctdb_traverse_async.
+ * return 0 - to continue traverse
+ * return 1 - to abort the traverse
+ *
+ * See Also:
+ *	ctdb_traverse_async()
+ */
+#define TRAVERSE_STATUS_RECORD		0
+#define TRAVERSE_STATUS_FINISHED	1
+#define TRAVERSE_STATUS_ERROR		2
+typedef int (*ctdb_traverse_callback_t)(struct ctdb_connection *ctdb,
+				    struct ctdb_db *ctdb_db,
+				    int status,
+				    TDB_DATA key,
+				    TDB_DATA data,
+				    void *private_data);
+
+/**
+ * ctdb_traverse_async - traverse a database.
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @callback: the callback once the record is locked (typesafe).
+ * @cbdata: the argument to callback()
+ *
+ * This returns true on success.
+ * when successfull, the callback will be invoked for each record
+ * until the traversal is finished.
+ *
+ * status == 
+ * TRAVERSE_STATUS_RECORD         key/data contains a record.
+ * TRAVERSE_STATUS_FINISHED       traverse is finished. key/data is undefined.
+ * TRAVERSE_STATUS_ERROR          an error occured during traverse.
+ *                                key/data is undefined.
+ *
+ * If failure is immediate, false is returned.
+ */
+bool ctdb_traverse_async(struct ctdb_db *ctdb_db,
+			 ctdb_traverse_callback_t callback, void *cbdata);
+
 /**
  * ctdb_message_fn_t - messaging callback for ctdb messages
  *
@@ -480,6 +521,7 @@ ctdb_getpublicips_send(struct ctdb_connection *ctdb,
 bool ctdb_getpublicips_recv(struct ctdb_connection *ctdb,
 		      struct ctdb_request *req, struct ctdb_all_public_ips **ips);
 
+
 /**
  * ctdb_getrecmaster_send - read the recovery master of a node
  * @ctdb: the ctdb_connection from ctdb_connect.
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index f62125c..31da0d5 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -838,24 +838,6 @@ int ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA
 int ctdb_control_writerecord(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
 
 
-struct ctdb_traverse_start {
-	uint32_t db_id;
-	uint32_t reqid;
-	uint64_t srvid;
-};
-
-/*
-  structure used to pass record data between the child and parent
- */
-struct ctdb_rec_data {
-	uint32_t length;
-	uint32_t reqid;
-	uint32_t keylen;
-	uint32_t datalen;
-	uint8_t  data[1];
-};
-				   
-
 /* structure used for pulldb control */
 struct ctdb_control_pulldb {
 	uint32_t db_id;
diff --git a/include/ctdb_protocol.h b/include/ctdb_protocol.h
index 1568460..baf1790 100644
--- a/include/ctdb_protocol.h
+++ b/include/ctdb_protocol.h
@@ -161,6 +161,9 @@ struct ctdb_call_info {
  */
 #define CTDB_SRVID_TEST_RANGE  0xFE03000000000000LL
 
+/* Range of ports reserved for traversals */
+#define CTDB_SRVID_TRAVERSE_RANGE  0xFE04000000000000LL
+
 /* used on the domain socket, send a pdu to the local daemon */
 #define CTDB_CURRENT_NODE     0xF0000001
 /* send a broadcast to all nodes in the cluster, active or not */
@@ -544,6 +547,23 @@ struct latency_counter {
 };
 
 /*
+  structure used to pass record data between the child and parent
+ */
+struct ctdb_rec_data {
+	uint32_t length;
+	uint32_t reqid;
+	uint32_t keylen;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+
+struct ctdb_traverse_start {
+	uint32_t db_id;
+	uint32_t reqid;
+	uint64_t srvid;
+};
+
+/*
   ctdb statistics information
  */
 struct ctdb_statistics {
diff --git a/libctdb/ctdb.c b/libctdb/ctdb.c
index e06c66c..7115982 100644
--- a/libctdb/ctdb.c
+++ b/libctdb/ctdb.c
@@ -936,3 +936,186 @@ bool ctdb_writerecord(struct ctdb_db *ctdb_db,
 		return false;
 	}
 }
+
+
+struct ctdb_traverse_state {
+	struct ctdb_request *handle;
+	struct ctdb_db *ctdb_db;
+	uint64_t srvid;
+
+	ctdb_traverse_callback_t callback;
+	void *cbdata;
+};
+
+static void traverse_remhnd_cb(struct ctdb_connection *ctdb,
+                        struct ctdb_request *req, void *private_data)
+{
+	struct ctdb_traverse_state *state = private_data;
+
+	if (!ctdb_remove_message_handler_recv(ctdb, state->handle)) {
+		DEBUG(ctdb, LOG_ERR,
+				"Failed to remove message handler for"
+				" traverse.");
+		state->callback(state->ctdb_db->ctdb, state->ctdb_db,
+				TRAVERSE_STATUS_ERROR,
+				tdb_null, tdb_null,
+				state->cbdata);
+	}
+	ctdb_request_free(ctdb, state->handle);
+	state->handle = NULL;
+	free(state);
+}
+	
+static void msg_h(struct ctdb_connection *ctdb, uint64_t srvid,
+	   TDB_DATA data, void *private_data)
+{
+	struct ctdb_traverse_state *state = private_data;
+	struct ctdb_db *ctdb_db = state->ctdb_db;
+	struct ctdb_rec_data *d = (struct ctdb_rec_data *)data.dptr;
+	TDB_DATA key;
+
+	if (data.dsize < sizeof(uint32_t) ||
+	    d->length != data.dsize) {
+		DEBUG(ctdb, LOG_ERR,
+			"Bad data size %u in traverse_handler",
+			(unsigned)data.dsize);
+		state->callback(state->ctdb_db->ctdb, state->ctdb_db,
+				TRAVERSE_STATUS_ERROR,
+				tdb_null, tdb_null,
+				state->cbdata);
+		state->handle = ctdb_remove_message_handler_send(
+				state->ctdb_db->ctdb, state->srvid,
+				msg_h, state,
+				traverse_remhnd_cb, state);
+		return;
+	}
+
+	key.dsize = d->keylen;
+	key.dptr  = &d->data[0];
+	data.dsize = d->datalen;
+	data.dptr = &d->data[d->keylen];
+
+	if (key.dsize == 0 && data.dsize == 0) {
+		state->callback(state->ctdb_db->ctdb, state->ctdb_db,
+				TRAVERSE_STATUS_FINISHED,
+				tdb_null, tdb_null,
+				state->cbdata);
+		state->handle = ctdb_remove_message_handler_send(
+				state->ctdb_db->ctdb, state->srvid,
+				msg_h, state,
+				traverse_remhnd_cb, state);
+		return;
+	}
+
+	if (data.dsize <= sizeof(struct ctdb_ltdb_header)) {
+		/* empty records are deleted records in ctdb */
+		return;
+	}
+
+	data.dsize -= sizeof(struct ctdb_ltdb_header);
+	data.dptr  += sizeof(struct ctdb_ltdb_header);
+
+	if (state->callback(ctdb, ctdb_db,
+			TRAVERSE_STATUS_RECORD,
+			key, data, state->cbdata) != 0) {
+		state->handle = ctdb_remove_message_handler_send(
+				state->ctdb_db->ctdb, state->srvid,
+				msg_h, state,
+				traverse_remhnd_cb, state);
+		return;
+	}
+}
+
+static void traverse_start_cb(struct ctdb_connection *ctdb,
+                        struct ctdb_request *req, void *private_data)
+{
+	struct ctdb_traverse_state *state = private_data;
+
+        ctdb_request_free(ctdb, state->handle);
+	state->handle = NULL;
+}
+
+static void traverse_msghnd_cb(struct ctdb_connection *ctdb,
+                        struct ctdb_request *req, void *private_data)
+{
+	struct ctdb_traverse_state *state = private_data;
+	struct ctdb_db *ctdb_db = state->ctdb_db;
+	struct ctdb_traverse_start t;
+
+	if (!ctdb_set_message_handler_recv(ctdb, state->handle)) {
+		DEBUG(ctdb, LOG_ERR,
+				"Failed to register message handler for"
+				" traverse.");
+		state->callback(state->ctdb_db->ctdb, state->ctdb_db,
+				TRAVERSE_STATUS_ERROR,
+				tdb_null, tdb_null,
+				state->cbdata);
+	        ctdb_request_free(ctdb, state->handle);
+		state->handle = NULL;
+		free(state);
+		return;
+        }
+        ctdb_request_free(ctdb, state->handle);
+	state->handle = NULL;
+
+	t.db_id = ctdb_db->id;
+	t.srvid = state->srvid;
+	t.reqid = 0;
+
+	state->handle = new_ctdb_control_request(ctdb,
+				CTDB_CONTROL_TRAVERSE_START,
+				CTDB_CURRENT_NODE,
+				&t, sizeof(t),
+				traverse_start_cb, state);
+	if (state->handle == NULL) {
+		DEBUG(ctdb, LOG_ERR,
+				"ctdb_traverse_async:"
+				" failed to send traverse_start control");
+		state->callback(state->ctdb_db->ctdb, state->ctdb_db,
+				TRAVERSE_STATUS_ERROR,
+				tdb_null, tdb_null,
+				state->cbdata);
+		state->handle = ctdb_remove_message_handler_send(
+				state->ctdb_db->ctdb, state->srvid,
+				msg_h, state,
+				traverse_remhnd_cb, state);
+		return;
+	}
+}
+
+bool ctdb_traverse_async(struct ctdb_db *ctdb_db,
+			 ctdb_traverse_callback_t callback, void *cbdata)
+{
+	struct ctdb_connection *ctdb = ctdb_db->ctdb;
+	struct ctdb_traverse_state *state;
+	static uint32_t tid = 0;
+
+	state = malloc(sizeof(struct ctdb_traverse_state));
+	if (state == NULL) {
+		DEBUG(ctdb, LOG_ERR,
+				"ctdb_traverse_async: no memory."
+				" allocate state failed");
+		return false;
+	}
+
+	tid++;
+	state->srvid = CTDB_SRVID_TRAVERSE_RANGE|tid;
+
+	state->callback = callback;
+	state->cbdata   = cbdata;
+	state->ctdb_db  = ctdb_db;
+
+	state->handle = ctdb_set_message_handler_send(ctdb_db->ctdb,
+				state->srvid,
+				msg_h, state,
+				traverse_msghnd_cb, state);
+	if (state->handle == NULL) {
+		DEBUG(ctdb, LOG_ERR,
+			"ctdb_traverse_async:"
+			" failed ctdb_set_message_handler_send");
+		free(state);
+		return false;
+	}
+
+	return true;
+}
diff --git a/libctdb/tst.c b/libctdb/tst.c
index e61561f..0e3531d 100644
--- a/libctdb/tst.c
+++ b/libctdb/tst.c
@@ -223,6 +223,25 @@ void message_handler_cb(struct ctdb_connection *ctdb,
 	registered = true;
 }
 
+static int traverse_callback(struct ctdb_connection *ctdb_connection, struct ctdb_db *ctdb_db, int status, TDB_DATA key, TDB_DATA data, void *private_data)
+{
+	if (status == TRAVERSE_STATUS_FINISHED) {
+		printf("Traverse finished\n");
+		return 0;
+	}
+	if (status == TRAVERSE_STATUS_ERROR) {
+		printf("Traverse failed\n");
+		return 1;
+	}
+
+	printf("traverse callback   status:%d\n", status);
+	printf("key: %d [%s]\n", key.dsize, key.dptr);
+	printf("data:%d [%s]\n", data.dsize, data.dptr);
+
+	return 0;
+}
+
+
 int main(int argc, char *argv[])
 {
 	struct ctdb_connection *ctdb_connection;
@@ -366,6 +385,9 @@ int main(int argc, char *argv[])
 	print_nodemap(nodemap);
 	ctdb_free_nodemap(nodemap);
 
+	printf("Traverse the test_test.tdb database\n");
+	ctdb_traverse_async(ctdb_db_context, traverse_callback, NULL);
+
 	for (;;) {
 
 	  pfd.events = ctdb_which_events(ctdb_connection);
diff --git a/server/ctdb_daemon.c b/server/ctdb_daemon.c
index 72c7293..362f1ce 100644
--- a/server/ctdb_daemon.c
+++ b/server/ctdb_daemon.c
@@ -43,6 +43,33 @@ static void print_exit_message(void)
 	DEBUG(DEBUG_NOTICE,("CTDB daemon shutting down\n"));
 }
 
+
+
+static void ctdb_time_tick(struct event_context *ev, struct timed_event *te, 
+				  struct timeval t, void *private_data)
+{
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+	if (getpid() != ctdbd_pid) {
+		return;
+	}
+
+	event_add_timed(ctdb->ev, ctdb, 
+			timeval_current_ofs(1, 0), 
+			ctdb_time_tick, ctdb);
+}
+
+/* Used to trigger a dummy event once per second, to make
+ * detection of hangs more reliable.
+ */
+static void ctdb_start_time_tickd(struct ctdb_context *ctdb)
+{
+	event_add_timed(ctdb->ev, ctdb, 
+			timeval_current_ofs(1, 0), 
+			ctdb_time_tick, ctdb);
+}
+
+
 /* called when the "startup" event script has finished */
 static void ctdb_start_transport(struct ctdb_context *ctdb)
 {
@@ -77,6 +104,9 @@ static void ctdb_start_transport(struct ctdb_context *ctdb)
 
 	/* start listening for recovery daemon pings */
 	ctdb_control_recd_ping(ctdb);
+
+	/* start listening to timer ticks */
+	ctdb_start_time_tickd(ctdb);
 }
 
 static void block_signal(int signum)
diff --git a/tools/ctdb.c b/tools/ctdb.c
index 51228a2..8d82d4d 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -1331,6 +1331,7 @@ static int move_ip(struct ctdb_context *ctdb, ctdb_sock_addr *addr, uint32_t pnn
 static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv)
 {
 	uint32_t pnn;
+	int ret, retries = 0;
 	ctdb_sock_addr addr;
 
 	if (argc < 2) {
@@ -1349,8 +1350,16 @@ static int control_moveip(struct ctdb_context *ctdb, int argc, const char **argv
 		return -1;
 	}
 
-	if (move_ip(ctdb, &addr, pnn) != 0) {
-		DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
+	do {
+		ret = move_ip(ctdb, &addr, pnn);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Wait 3 second and try again.\n", pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Giving up.\n", pnn));
 		return -1;
 	}
 
@@ -1657,7 +1666,7 @@ again:
 static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 {
 	int i, ret;
-	int len;
+	int len, retries = 0;
 	uint32_t pnn;
 	unsigned mask;
 	ctdb_sock_addr addr;
@@ -1703,9 +1712,16 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 	pub->len   = strlen(argv[1])+1;
 	memcpy(&pub->iface[0], argv[1], strlen(argv[1])+1);
 
-	ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
+	do {
+		ret = ctdb_ctrl_add_public_ip(ctdb, TIMELIMIT(), options.pnn, pub);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Wait 3 seconds and try again.\n", options.pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
 	if (ret != 0) {
-		DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u\n", options.pnn));
+		DEBUG(DEBUG_ERR, ("Unable to add public ip to node %u. Giving up.\n", options.pnn));
 		talloc_free(tmp_ctx);
 		return ret;
 	}
@@ -1717,14 +1733,31 @@ static int control_addip(struct ctdb_context *ctdb, int argc, const char **argv)
 		pnn  = ips->ips[i].pnn;
 	}
 
-	if (move_ip(ctdb, &addr, pnn) != 0) {
-		DEBUG(DEBUG_ERR,("Failed to move ip to node %d\n", pnn));
-		return -1;
+	do {
+		ret = move_ip(ctdb, &addr, pnn);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR,("Failed to move ip to node %d. wait 3 seconds and try again.\n", pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR,("Failed to move ip to node %d. Giving up.\n", pnn));
+		talloc_free(tmp_ctx);
+		return ret;
 	}
 
-	ret = control_ipreallocate(ctdb, argc, argv);
+	do {
+		ret = control_ipreallocate(ctdb, argc, argv);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Wait 3 seconds and try again.\n", options.pnn));
+			sleep(3);
+			retries++;
+		}
+	} while (retries < 5 && ret != 0);
 	if (ret != 0) {
-		DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u\n", options.pnn));
+		DEBUG(DEBUG_ERR, ("IP Reallocate failed on node %u. Giving up.\n", options.pnn));
+		talloc_free(tmp_ctx);
 		return ret;


-- 
CTDB repository


More information about the samba-cvs mailing list