[PATCH] cleanups around ctdbd_conn.c

Volker Lendecke Volker.Lendecke at SerNet.DE
Tue May 26 14:13:03 MDT 2015


Hi!

Review&push appreciated!

Thanks,

Volker

-- 
SerNet GmbH, Bahnhofsallee 1b, 37081 Göttingen
phone: +49-551-370000-0, fax: +49-551-370000-9
AG Göttingen, HRB 2816, GF: Dr. Johannes Loxen
http://www.sernet.de, mailto:kontakt at sernet.de
-------------- next part --------------
From db5b5407c3a9265ec65303f88f5ed49226de2f3c Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Sun, 17 May 2015 20:23:35 +0200
Subject: [PATCH 01/15] ctdbd_conn: Use read_data()

This is a much smaller dependency than read_data_ntstatus

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c |   22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 1285e4b..f5044e1 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -343,7 +343,7 @@ static NTSTATUS ctdb_read_packet(int fd, TALLOC_CTX *mem_ctx,
 	struct ctdb_req_header *req;
 	int ret, revents;
 	uint32_t msglen;
-	NTSTATUS status;
+	ssize_t nread;
 
 	if (timeout == 0) {
 		timeout = -1;
@@ -362,9 +362,12 @@ static NTSTATUS ctdb_read_packet(int fd, TALLOC_CTX *mem_ctx,
 		}
 	}
 
-	status = read_data_ntstatus(fd, (char *)&msglen, sizeof(msglen));
-	if (!NT_STATUS_IS_OK(status)) {
-		return status;
+	nread = read_data(fd, &msglen, sizeof(msglen));
+	if (nread == -1) {
+		return map_nt_error_from_unix(errno);
+	}
+	if (nread == 0) {
+		return NT_STATUS_UNEXPECTED_IO_ERROR;
 	}
 
 	if (msglen < sizeof(struct ctdb_req_header)) {
@@ -379,10 +382,13 @@ static NTSTATUS ctdb_read_packet(int fd, TALLOC_CTX *mem_ctx,
 
 	req->length = msglen;
 
-	status = read_data_ntstatus(fd, ((char *)req) + sizeof(msglen),
-				    msglen - sizeof(msglen));
-	if (!NT_STATUS_IS_OK(status)) {
-		return status;
+	nread = read_data(fd, ((char *)req) + sizeof(msglen),
+			  msglen - sizeof(msglen));
+	if (nread == -1) {
+		return map_nt_error_from_unix(errno);
+	}
+	if (nread == 0) {
+		return NT_STATUS_UNEXPECTED_IO_ERROR;
 	}
 
 	*result = req;
-- 
1.7.9.5


From 050fc90d742fc8915172cb9fa3b0291aca359606 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Mon, 18 May 2015 13:15:35 +0200
Subject: [PATCH 02/15] messaging: Register a tevent context for secondary dgm
 refs

When a secondary messaging context is initialized, we need to register
its tevent context with the lower level dgm context. Seen when using
smbstatus with clustering.

The TALLOC_FREE(r->tevent_handle) in the destructor might not be
necessary, but I want to free the tevent reference before
the context goes away.

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/messages_dgm_ref.c |   12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/source3/lib/messages_dgm_ref.c b/source3/lib/messages_dgm_ref.c
index 32b9c98..b4511e0 100644
--- a/source3/lib/messages_dgm_ref.c
+++ b/source3/lib/messages_dgm_ref.c
@@ -26,6 +26,7 @@
 
 struct msg_dgm_ref {
 	struct msg_dgm_ref *prev, *next;
+	void *tevent_handle;
 	void (*recv_cb)(const uint8_t *msg, size_t msg_len,
 			int *fds, size_t num_fds, void *private_data);
 	void *recv_cb_private_data;
@@ -55,6 +56,7 @@ void *messaging_dgm_ref(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
 		*err = ENOMEM;
 		return NULL;
 	}
+	result->tevent_handle = NULL;
 
 	tmp_refs = refs;
 
@@ -79,6 +81,14 @@ void *messaging_dgm_ref(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
 			return NULL;
 		}
 		dgm_pid = getpid();
+	} else {
+		result->tevent_handle = messaging_dgm_register_tevent_context(
+			result, ev);
+		if (result->tevent_handle == NULL) {
+			TALLOC_FREE(result);
+			*err = ENOMEM;
+			return NULL;
+		}
 	}
 
 	refs = tmp_refs;
@@ -114,6 +124,8 @@ static int msg_dgm_ref_destructor(struct msg_dgm_ref *r)
 	}
 	DLIST_REMOVE(refs, r);
 
+	TALLOC_FREE(r->tevent_handle);
+
 	if (refs == NULL) {
 		messaging_dgm_destroy();
 	}
-- 
1.7.9.5


From 2679347f027600a72b46bb28e620ef056a59c649 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Tue, 19 May 2015 07:01:55 +0200
Subject: [PATCH 03/15] ctdbd_conn: Add callback fields for registered
 serverids

The idea is to move message handling out of ctdbd_conn

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c |   33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index f5044e1..cdec978 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -32,12 +32,18 @@
 #include "ctdb.h"
 #include "ctdb_private.h"
 
+struct ctdbd_srvid_cb {
+	uint64_t srvid;
+	void (*cb)(struct ctdb_req_message *msg, void *private_data);
+	void *private_data;
+};
+
 struct ctdbd_connection {
 	struct messaging_context *msg_ctx;
 	uint32_t reqid;
 	uint32_t our_vnn;
 	uint64_t rand_srvid;
-	uint64_t *srvids;
+	struct ctdbd_srvid_cb *callbacks;
 	int fd;
 	struct tevent_fd *fde;
 
@@ -95,8 +101,8 @@ NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid)
 
 	NTSTATUS status;
 	int cstatus;
-	size_t num_srvids;
-	uint64_t *tmp;
+	size_t num_callbacks;
+	struct ctdbd_srvid_cb *tmp;
 
 	status = ctdbd_control(conn, CTDB_CURRENT_NODE,
 			       CTDB_CONTROL_REGISTER_SRVID, srvid, 0,
@@ -105,27 +111,30 @@ NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid)
 		return status;
 	}
 
-	num_srvids = talloc_array_length(conn->srvids);
+	num_callbacks = talloc_array_length(conn->callbacks);
 
-	tmp = talloc_realloc(conn, conn->srvids, uint64_t,
-			     num_srvids + 1);
+	tmp = talloc_realloc(conn, conn->callbacks, struct ctdbd_srvid_cb,
+			     num_callbacks + 1);
 	if (tmp == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
-	conn->srvids = tmp;
+	conn->callbacks = tmp;
+
+	conn->callbacks[num_callbacks] = (struct ctdbd_srvid_cb) {
+		.srvid = srvid
+	};
 
-	conn->srvids[num_srvids] = srvid;
 	return NT_STATUS_OK;
 }
 
 static bool ctdb_is_our_srvid(struct ctdbd_connection *conn, uint64_t srvid)
 {
-	size_t i, num_srvids;
+	size_t i, num_callbacks;
 
-	num_srvids = talloc_array_length(conn->srvids);
+	num_callbacks = talloc_array_length(conn->callbacks);
 
-	for (i=0; i<num_srvids; i++) {
-		if (srvid == conn->srvids[i]) {
+	for (i=0; i<num_callbacks; i++) {
+		if (srvid == conn->callbacks[i].srvid) {
 			return true;
 		}
 	}
-- 
1.7.9.5


From 890d9419446bab529a0a7c00e7d038b884064278 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Tue, 19 May 2015 07:05:24 +0200
Subject: [PATCH 04/15] ctdbd_conn: Add callback args to register_with_ctdbd

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/include/ctdbd_conn.h |    8 +++++++-
 source3/lib/ctdb_dummy.c     |    5 ++++-
 source3/lib/ctdbd_conn.c     |   20 ++++++++++++--------
 source3/lib/serverid.c       |    3 ++-
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/source3/include/ctdbd_conn.h b/source3/include/ctdbd_conn.h
index 36932bd..397c093 100644
--- a/source3/include/ctdbd_conn.h
+++ b/source3/include/ctdbd_conn.h
@@ -90,7 +90,13 @@ NTSTATUS ctdbd_control_local(struct ctdbd_connection *conn, uint32_t opcode,
 			     int *cstatus);
 NTSTATUS ctdb_watch_us(struct ctdbd_connection *conn);
 NTSTATUS ctdb_unwatch(struct ctdbd_connection *conn);
-NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid);
+
+struct ctdb_req_message;
+
+NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid,
+			     void (*cb)(struct ctdb_req_message *msg,
+					void *private_data),
+			     void *private_data);
 NTSTATUS ctdbd_probe(void);
 
 #endif /* _CTDBD_CONN_H */
diff --git a/source3/lib/ctdb_dummy.c b/source3/lib/ctdb_dummy.c
index e19765b..ca1309b 100644
--- a/source3/lib/ctdb_dummy.c
+++ b/source3/lib/ctdb_dummy.c
@@ -44,7 +44,10 @@ NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid)
+NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid,
+			     void (*cb)(struct ctdb_req_message *msg,
+					void *private_data),
+			     void *private_data)
 {
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index cdec978..9794d3e 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -96,7 +96,10 @@ static void ctdb_packet_dump(struct ctdb_req_header *hdr)
 /*
  * Register a srvid with ctdbd
  */
-NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid)
+NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid,
+			     void (*cb)(struct ctdb_req_message *msg,
+					void *private_data),
+			     void *private_data)
 {
 
 	NTSTATUS status;
@@ -121,7 +124,7 @@ NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid)
 	conn->callbacks = tmp;
 
 	conn->callbacks[num_callbacks] = (struct ctdbd_srvid_cb) {
-		.srvid = srvid
+		.srvid = srvid, .cb = cb, .private_data = private_data
 	};
 
 	return NT_STATUS_OK;
@@ -573,7 +576,7 @@ static NTSTATUS ctdbd_init_connection(TALLOC_CTX *mem_ctx,
 	generate_random_buffer((unsigned char *)&conn->rand_srvid,
 			       sizeof(conn->rand_srvid));
 
-	status = register_with_ctdbd(conn, conn->rand_srvid);
+	status = register_with_ctdbd(conn, conn->rand_srvid, NULL, NULL);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(5, ("Could not register random srvid: %s\n",
@@ -605,17 +608,18 @@ NTSTATUS ctdbd_messaging_connection(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	status = register_with_ctdbd(conn, (uint64_t)getpid());
+	status = register_with_ctdbd(conn, (uint64_t)getpid(), NULL, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto fail;
 	}
 
-	status = register_with_ctdbd(conn, MSG_SRVID_SAMBA);
+	status = register_with_ctdbd(conn, MSG_SRVID_SAMBA, NULL, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto fail;
 	}
 
-	status = register_with_ctdbd(conn, CTDB_SRVID_SAMBA_NOTIFY);
+	status = register_with_ctdbd(conn, CTDB_SRVID_SAMBA_NOTIFY,
+				     NULL, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto fail;
 	}
@@ -1685,7 +1689,7 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 	 * We want to be told about IP releases
 	 */
 
-	status = register_with_ctdbd(conn, CTDB_SRVID_RELEASE_IP);
+	status = register_with_ctdbd(conn, CTDB_SRVID_RELEASE_IP, NULL, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		return status;
 	}
@@ -1705,7 +1709,7 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
  */
 NTSTATUS ctdbd_register_reconfigure(struct ctdbd_connection *conn)
 {
-	return register_with_ctdbd(conn, CTDB_SRVID_RECONFIGURE);
+	return register_with_ctdbd(conn, CTDB_SRVID_RECONFIGURE, NULL, NULL);
 }
 
 /*
diff --git a/source3/lib/serverid.c b/source3/lib/serverid.c
index f5e0937..39c733c 100644
--- a/source3/lib/serverid.c
+++ b/source3/lib/serverid.c
@@ -123,7 +123,8 @@ bool serverid_register(const struct server_id id, uint32_t msg_flags)
 	if (lp_clustering() &&
 	    ctdb_serverids_exist_supported(messaging_ctdbd_connection()))
 	{
-		register_with_ctdbd(messaging_ctdbd_connection(), id.unique_id);
+		register_with_ctdbd(messaging_ctdbd_connection(), id.unique_id,
+				    NULL, NULL);
 	}
 
 	ret = true;
-- 
1.7.9.5


From 8be93dacad2e9f0bc2967690062bc7bc3784ce3a Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Tue, 19 May 2015 15:07:33 +0200
Subject: [PATCH 05/15] ctdbd_conn: Call back when we get a msg

This activates the callbacks just added

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c |   20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 9794d3e..a0dbd32 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -144,6 +144,22 @@ static bool ctdb_is_our_srvid(struct ctdbd_connection *conn, uint64_t srvid)
 	return false;
 }
 
+static void ctdbd_msg_call_back(struct ctdbd_connection *conn,
+				struct ctdb_req_message *msg)
+{
+	size_t i, num_callbacks;
+
+	num_callbacks = talloc_array_length(conn->callbacks);
+
+	for (i=0; i<num_callbacks; i++) {
+		struct ctdbd_srvid_cb *cb = &conn->callbacks[i];
+
+		if ((cb->srvid == msg->srvid) && (cb->cb != NULL)) {
+			cb->cb(msg, cb->private_data);
+		}
+	}
+}
+
 /*
  * get our vnn from the cluster
  */
@@ -480,6 +496,8 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32_t reqid,
 			goto next_pkt;
 		}
 
+		ctdbd_msg_call_back(conn, msg);
+
 		msg_state = talloc(NULL, struct deferred_msg_state);
 		if (msg_state == NULL) {
 			DEBUG(0, ("talloc failed\n"));
@@ -697,6 +715,8 @@ static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
 		return NT_STATUS_OK;
 	}
 
+	ctdbd_msg_call_back(conn, msg);
+
 	if (!ctdb_is_our_srvid(conn, msg->srvid)) {
 		DEBUG(0,("Got unexpected message with srvid=%llu\n",
 			 (unsigned long long)msg->srvid));
-- 
1.7.9.5


From 1cd1dc3c03a1d57fe870b7aaccb8be5257d6eb28 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Tue, 19 May 2015 16:55:32 +0200
Subject: [PATCH 06/15] ctdbd_conn: Move message handling out of ctdbd_conn.c

This also removes the deferred message handling. It's no longer required,
because the messaging_send_iov_from always goes through the kernel which
takes at least one round through tevent.

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c     |  141 ------------------------------------------
 source3/lib/messages_ctdbd.c |   85 +++++++++++++++++++++++++
 2 files changed, 85 insertions(+), 141 deletions(-)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index a0dbd32..4e03018 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -130,20 +130,6 @@ NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid,
 	return NT_STATUS_OK;
 }
 
-static bool ctdb_is_our_srvid(struct ctdbd_connection *conn, uint64_t srvid)
-{
-	size_t i, num_callbacks;
-
-	num_callbacks = talloc_array_length(conn->callbacks);
-
-	for (i=0; i<num_callbacks; i++) {
-		if (srvid == conn->callbacks[i].srvid) {
-			return true;
-		}
-	}
-	return false;
-}
-
 static void ctdbd_msg_call_back(struct ctdbd_connection *conn,
 				struct ctdb_req_message *msg)
 {
@@ -292,78 +278,6 @@ static int ctdbd_connect(int *pfd)
 	return 0;
 }
 
-/*
- * State necessary to defer an incoming message while we are waiting for a
- * ctdb reply.
- */
-
-struct deferred_msg_state {
-	struct messaging_context *msg_ctx;
-	struct messaging_rec *rec;
-};
-
-/*
- * Timed event handler for the deferred message
- */
-
-static void deferred_message_dispatch(struct tevent_context *event_ctx,
-				      struct tevent_timer *te,
-				      struct timeval now,
-				      void *private_data)
-{
-	struct deferred_msg_state *state = talloc_get_type_abort(
-		private_data, struct deferred_msg_state);
-
-	messaging_dispatch_rec(state->msg_ctx, state->rec);
-	TALLOC_FREE(state);
-	TALLOC_FREE(te);
-}
-
-/*
- * Fetch a messaging_rec from an incoming ctdb style message
- */
-
-static struct messaging_rec *ctdb_pull_messaging_rec(TALLOC_CTX *mem_ctx,
-						     size_t overall_length,
-						     struct ctdb_req_message *msg)
-{
-	struct messaging_rec *result;
-	DATA_BLOB blob;
-	enum ndr_err_code ndr_err;
-
-	if ((overall_length < offsetof(struct ctdb_req_message, data))
-	    || (overall_length
-		< offsetof(struct ctdb_req_message, data) + msg->datalen)) {
-
-		cluster_fatal("got invalid msg length");
-	}
-
-	if (!(result = talloc(mem_ctx, struct messaging_rec))) {
-		DEBUG(0, ("talloc failed\n"));
-		return NULL;
-	}
-
-	blob = data_blob_const(msg->data, msg->datalen);
-
-	ndr_err = ndr_pull_struct_blob(
-		&blob, result, result,
-		(ndr_pull_flags_fn_t)ndr_pull_messaging_rec);
-
-	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-		DEBUG(0, ("ndr_pull_struct_blob failed: %s\n",
-			  ndr_errstr(ndr_err)));
-		TALLOC_FREE(result);
-		return NULL;
-	}
-
-	if (DEBUGLEVEL >= 11) {
-		DEBUG(11, ("ctdb_pull_messaging_rec:\n"));
-		NDR_PRINT_DEBUG(messaging_rec, result);
-	}
-
-	return result;
-}
-
 static NTSTATUS ctdb_read_packet(int fd, TALLOC_CTX *mem_ctx,
 				 struct ctdb_req_header **result)
 {
@@ -447,8 +361,6 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32_t reqid,
 	ctdb_packet_dump(hdr);
 
 	if (hdr->operation == CTDB_REQ_MESSAGE) {
-		struct tevent_timer *evt;
-		struct deferred_msg_state *msg_state;
 		struct ctdb_req_message *msg = (struct ctdb_req_message *)hdr;
 
 		if (conn->msg_ctx == NULL) {
@@ -497,42 +409,7 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32_t reqid,
 		}
 
 		ctdbd_msg_call_back(conn, msg);
-
-		msg_state = talloc(NULL, struct deferred_msg_state);
-		if (msg_state == NULL) {
-			DEBUG(0, ("talloc failed\n"));
-			TALLOC_FREE(hdr);
-			goto next_pkt;
-		}
-
-		if (!(msg_state->rec = ctdb_pull_messaging_rec(
-			      msg_state, msg->hdr.length, msg))) {
-			DEBUG(0, ("ctdbd_pull_messaging_rec failed\n"));
-			TALLOC_FREE(msg_state);
-			TALLOC_FREE(hdr);
-			goto next_pkt;
-		}
-
 		TALLOC_FREE(hdr);
-
-		msg_state->msg_ctx = conn->msg_ctx;
-
-		/*
-		 * We're waiting for a call reply, but an async message has
-		 * crossed. Defer dispatching to the toplevel event loop.
-		 */
-		evt = tevent_add_timer(messaging_tevent_context(conn->msg_ctx),
-				      messaging_tevent_context(conn->msg_ctx),
-				      timeval_zero(),
-				      deferred_message_dispatch,
-				      msg_state);
-		if (evt == NULL) {
-			DEBUG(0, ("event_add_timed failed\n"));
-			TALLOC_FREE(msg_state);
-			TALLOC_FREE(hdr);
-			goto next_pkt;
-		}
-
 		goto next_pkt;
 	}
 
@@ -626,11 +503,6 @@ NTSTATUS ctdbd_messaging_connection(TALLOC_CTX *mem_ctx,
 		return status;
 	}
 
-	status = register_with_ctdbd(conn, (uint64_t)getpid(), NULL, NULL);
-	if (!NT_STATUS_IS_OK(status)) {
-		goto fail;
-	}
-
 	status = register_with_ctdbd(conn, MSG_SRVID_SAMBA, NULL, NULL);
 	if (!NT_STATUS_IS_OK(status)) {
 		goto fail;
@@ -668,7 +540,6 @@ static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
 				    struct ctdb_req_header *hdr)
 {
 	struct ctdb_req_message *msg;
-	struct messaging_rec *msg_rec;
 
 	if (hdr->operation != CTDB_REQ_MESSAGE) {
 		DEBUG(0, ("Received async msg of type %u, discarding\n",
@@ -717,18 +588,6 @@ static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
 
 	ctdbd_msg_call_back(conn, msg);
 
-	if (!ctdb_is_our_srvid(conn, msg->srvid)) {
-		DEBUG(0,("Got unexpected message with srvid=%llu\n",
-			 (unsigned long long)msg->srvid));
-		return NT_STATUS_OK;
-	}
-
-	msg_rec = ctdb_pull_messaging_rec(talloc_tos(), msg->hdr.length, msg);
-	if (msg_rec == NULL) {
-		DEBUG(10, ("ctdb_pull_messaging_rec failed\n"));
-		return NT_STATUS_NO_MEMORY;
-	}
-	messaging_dispatch_rec(conn->msg_ctx, msg_rec);
 	return NT_STATUS_OK;
 }
 
diff --git a/source3/lib/messages_ctdbd.c b/source3/lib/messages_ctdbd.c
index 799780e..430dc51 100644
--- a/source3/lib/messages_ctdbd.c
+++ b/source3/lib/messages_ctdbd.c
@@ -128,6 +128,88 @@ static int messaging_ctdbd_destructor(struct messaging_ctdbd_context *ctx)
 	return 0;
 }
 
+static struct messaging_rec *ctdb_pull_messaging_rec(
+	TALLOC_CTX *mem_ctx, const struct ctdb_req_message *msg)
+{
+	struct messaging_rec *result;
+	DATA_BLOB blob;
+	enum ndr_err_code ndr_err;
+	size_t len = msg->hdr.length;
+
+	if (len < offsetof(struct ctdb_req_message, data)) {
+		return NULL;
+	}
+	len -= offsetof(struct ctdb_req_message, data);
+
+	if (len < msg->datalen) {
+		return NULL;
+	}
+
+	result = talloc(mem_ctx, struct messaging_rec);
+	if (result == NULL) {
+		return NULL;
+	}
+
+	blob = data_blob_const(msg->data, msg->datalen);
+
+	ndr_err = ndr_pull_struct_blob_all(
+		&blob, result, result,
+		(ndr_pull_flags_fn_t)ndr_pull_messaging_rec);
+
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		DEBUG(0, ("ndr_pull_struct_blob failed: %s\n",
+			  ndr_errstr(ndr_err)));
+		TALLOC_FREE(result);
+		return NULL;
+	}
+
+	if (DEBUGLEVEL >= 11) {
+		DEBUG(11, ("ctdb_pull_messaging_rec:\n"));
+		NDR_PRINT_DEBUG(messaging_rec, result);
+	}
+
+	return result;
+}
+
+static void messaging_ctdb_recv(struct ctdb_req_message *msg,
+				void *private_data)
+{
+	struct messaging_context *msg_ctx = talloc_get_type_abort(
+		private_data, struct messaging_context);
+	struct server_id me = messaging_server_id(msg_ctx);
+	struct messaging_rec *rec;
+	NTSTATUS status;
+	struct iovec iov;
+
+	rec = ctdb_pull_messaging_rec(msg_ctx, msg);
+	if (rec == NULL) {
+		DEBUG(10, ("%s: ctdb_pull_messaging_rec failed\n", __func__));
+		return;
+	}
+
+	if (!server_id_same_process(&me, &rec->dest)) {
+		struct server_id_buf id1, id2;
+
+		DEBUG(10, ("%s: I'm %s, ignoring msg to %s\n", __func__,
+			   server_id_str_buf(me, &id1),
+			   server_id_str_buf(rec->dest, &id2)));
+		TALLOC_FREE(rec);
+		return;
+	}
+
+	iov = (struct iovec) { .iov_base = rec->buf.data,
+			       .iov_len = rec->buf.length };
+
+	status = messaging_send_iov_from(msg_ctx, rec->src, rec->dest,
+					 rec->msg_type, &iov, 1, NULL, 0);
+	TALLOC_FREE(rec);
+
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(10, ("%s: messaging_send_iov_from failed: %s\n",
+			   __func__, nt_errstr(status)));
+	}
+}
+
 NTSTATUS messaging_ctdbd_init(struct messaging_context *msg_ctx,
 			      TALLOC_CTX *mem_ctx,
 			      struct messaging_backend **presult)
@@ -165,6 +247,9 @@ NTSTATUS messaging_ctdbd_init(struct messaging_context *msg_ctx,
 		return status;
 	}
 
+	status = register_with_ctdbd(ctx->conn, getpid(),
+				     messaging_ctdb_recv, msg_ctx);
+
 	global_ctdb_connection_pid = getpid();
 	global_ctdbd_connection = ctx->conn;
 	talloc_set_destructor(ctx, messaging_ctdbd_destructor);
-- 
1.7.9.5


From c115046d6d3464733b4b6a5744e2b5404c6d2dc2 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Tue, 19 May 2015 22:15:57 +0200
Subject: [PATCH 07/15] ctdbd_conn: Move ctdbd_register_reconfigure to a
 callback

Move functionality out of ctdbd_conn to its right place into smbd

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/include/ctdbd_conn.h |    2 --
 source3/lib/ctdb_dummy.c     |    5 -----
 source3/lib/ctdbd_conn.c     |   37 -------------------------------------
 source3/smbd/server.c        |   25 ++++++++++++++++++++++++-
 4 files changed, 24 insertions(+), 45 deletions(-)

diff --git a/source3/include/ctdbd_conn.h b/source3/include/ctdbd_conn.h
index 397c093..13889fc 100644
--- a/source3/include/ctdbd_conn.h
+++ b/source3/include/ctdbd_conn.h
@@ -82,8 +82,6 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 						       void *private_data),
 			    void *private_data);
 
-NTSTATUS ctdbd_register_reconfigure(struct ctdbd_connection *conn);
-
 NTSTATUS ctdbd_control_local(struct ctdbd_connection *conn, uint32_t opcode,
 			     uint64_t srvid, uint32_t flags, TDB_DATA data,
 			     TALLOC_CTX *mem_ctx, TDB_DATA *outdata,
diff --git a/source3/lib/ctdb_dummy.c b/source3/lib/ctdb_dummy.c
index ca1309b..50de9ad 100644
--- a/source3/lib/ctdb_dummy.c
+++ b/source3/lib/ctdb_dummy.c
@@ -52,11 +52,6 @@ NTSTATUS register_with_ctdbd(struct ctdbd_connection *conn, uint64_t srvid,
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-NTSTATUS ctdbd_register_reconfigure(struct ctdbd_connection *conn)
-{
-	return NT_STATUS_NOT_IMPLEMENTED;
-}
-
 NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 			    const struct sockaddr_storage *_server,
 			    const struct sockaddr_storage *_client,
diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 4e03018..001eb68 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -394,20 +394,6 @@ static NTSTATUS ctdb_read_req(struct ctdbd_connection *conn, uint32_t reqid,
 			goto next_pkt;
 		}
 
-		if ((msg->srvid == CTDB_SRVID_RECONFIGURE)
-		    || (msg->srvid == CTDB_SRVID_SAMBA_NOTIFY)) {
-
-			DEBUG(1, ("ctdb_read_req: Got %s message\n",
-				  (msg->srvid == CTDB_SRVID_RECONFIGURE)
-				  ? "cluster reconfigure" : "SAMBA_NOTIFY"));
-
-			messaging_send(conn->msg_ctx,
-				       messaging_server_id(conn->msg_ctx),
-				       MSG_SMB_BRL_VALIDATE, &data_blob_null);
-			TALLOC_FREE(hdr);
-			goto next_pkt;
-		}
-
 		ctdbd_msg_call_back(conn, msg);
 		TALLOC_FREE(hdr);
 		goto next_pkt;
@@ -571,21 +557,6 @@ static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
 
 	SMB_ASSERT(conn->msg_ctx != NULL);
 
-	if ((msg->srvid == CTDB_SRVID_RECONFIGURE)
-	    || (msg->srvid == CTDB_SRVID_SAMBA_NOTIFY)){
-		DEBUG(0,("Got cluster reconfigure message\n"));
-		/*
-		 * when the cluster is reconfigured or someone of the
-		 * family has passed away (SAMBA_NOTIFY), we need to
-		 * clean the brl database
-		 */
-		messaging_send(conn->msg_ctx,
-			       messaging_server_id(conn->msg_ctx),
-			       MSG_SMB_BRL_VALIDATE, &data_blob_null);
-
-		return NT_STATUS_OK;
-	}
-
 	ctdbd_msg_call_back(conn, msg);
 
 	return NT_STATUS_OK;
@@ -1584,14 +1555,6 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 }
 
 /*
- * We want to handle reconfigure events
- */
-NTSTATUS ctdbd_register_reconfigure(struct ctdbd_connection *conn)
-{
-	return register_with_ctdbd(conn, CTDB_SRVID_RECONFIGURE, NULL, NULL);
-}
-
-/*
   call a control on the local node
  */
 NTSTATUS ctdbd_control_local(struct ctdbd_connection *conn, uint32_t opcode,
diff --git a/source3/smbd/server.c b/source3/smbd/server.c
index 1833462..9746d84 100644
--- a/source3/smbd/server.c
+++ b/source3/smbd/server.c
@@ -48,6 +48,7 @@
 #include "lib/smbd_shim.h"
 #include "scavenger.h"
 #include "locking/leases_db.h"
+#include "../../ctdb/include/ctdb_protocol.h"
 
 struct smbd_open_socket;
 struct smbd_child_pid;
@@ -260,6 +261,23 @@ static void smbd_parent_id_cache_delete(struct messaging_context *ctx,
 	messaging_send_to_children(ctx, msg_type, msg_data);
 }
 
+static void smbd_parent_ctdb_reconfigured(struct ctdb_req_message *msg,
+					  void *private_data)
+{
+	struct messaging_context *msg_ctx = talloc_get_type_abort(
+		private_data, struct messaging_context);
+
+	DEBUG(10, ("Got %s message\n", (msg->srvid == CTDB_SRVID_RECONFIGURE)
+		   ? "cluster reconfigure" : "SAMBA_NOTIFY"));
+
+	/*
+	 * Someone from the family died, validate our locks
+	 */
+
+	messaging_send_buf(msg_ctx, messaging_server_id(msg_ctx),
+			   MSG_SMB_BRL_VALIDATE, NULL, 0);
+}
+
 struct smbd_parent_notify_state {
 	struct tevent_context *ev;
 	struct messaging_context *msg;
@@ -898,7 +916,12 @@ static bool open_sockets_smbd(struct smbd_parent_context *parent,
 			   ID_CACHE_KILL, smbd_parent_id_cache_kill);
 
 	if (lp_clustering()) {
-		ctdbd_register_reconfigure(messaging_ctdbd_connection());
+		struct ctdbd_connection *conn = messaging_ctdbd_connection();
+
+		register_with_ctdbd(conn, CTDB_SRVID_RECONFIGURE,
+				    smbd_parent_ctdb_reconfigured, msg_ctx);
+		register_with_ctdbd(conn, CTDB_SRVID_SAMBA_NOTIFY,
+				    smbd_parent_ctdb_reconfigured, msg_ctx);
 	}
 
 #ifdef DEVELOPER
-- 
1.7.9.5


From 1ff3d82deedfb54a69715c2b96da9298aa22b027 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Wed, 20 May 2015 08:12:46 +0200
Subject: [PATCH 08/15] ctdbd_conn: simplify ctdbd_register_ips

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c |    6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 001eb68..f730aaf 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -1502,7 +1502,7 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 			    void *private_data)
 {
 	struct ctdb_control_tcp_addr p;
-	TDB_DATA data;
+	TDB_DATA data = { .dptr = (uint8_t *)&p, .dsize = sizeof(p) };
 	NTSTATUS status;
 	struct sockaddr_storage client;
 	struct sockaddr_storage server;
@@ -1519,14 +1519,10 @@ NTSTATUS ctdbd_register_ips(struct ctdbd_connection *conn,
 	case AF_INET:
 		memcpy(&p.dest.ip, &server, sizeof(p.dest.ip));
 		memcpy(&p.src.ip, &client, sizeof(p.src.ip));
-		data.dptr = (uint8_t *)&p;
-		data.dsize = sizeof(p);
 		break;
 	case AF_INET6:
 		memcpy(&p.dest.ip6, &server, sizeof(p.dest.ip6));
 		memcpy(&p.src.ip6, &client, sizeof(p.src.ip6));
-		data.dptr = (uint8_t *)&p;
-		data.dsize = sizeof(p);
 		break;
 	default:
 		return NT_STATUS_INTERNAL_ERROR;
-- 
1.7.9.5


From 5b2477274cc4e3eadffb21331cc37160059b9bd8 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Wed, 20 May 2015 11:17:25 +0200
Subject: [PATCH 09/15] ctdbd_conn: Move ndr marshalling to messages_ctdb.c

The inter-node message format belongs into messages_ctdb, not into the
generic ctdb connection layer

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/include/ctdbd_conn.h |    3 ---
 source3/lib/ctdb_dummy.c     |    7 -------
 source3/lib/ctdbd_conn.c     |   28 ----------------------------
 source3/lib/messages_ctdbd.c |   15 ++++++++++++++-
 4 files changed, 14 insertions(+), 39 deletions(-)

diff --git a/source3/include/ctdbd_conn.h b/source3/include/ctdbd_conn.h
index 13889fc..6b3ebf1 100644
--- a/source3/include/ctdbd_conn.h
+++ b/source3/include/ctdbd_conn.h
@@ -38,9 +38,6 @@ struct messaging_context *ctdb_conn_msg_ctx(struct ctdbd_connection *conn);
 
 int ctdbd_conn_get_fd(struct ctdbd_connection *conn);
 
-NTSTATUS ctdbd_messaging_send(struct ctdbd_connection *conn,
-			      uint32_t dst_vnn, uint64_t dst_srvid,
-			      struct messaging_rec *msg);
 NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
 				   uint32_t dst_vnn, uint64_t dst_srvid,
 				   const uint8_t *buf, size_t buflen);
diff --git a/source3/lib/ctdb_dummy.c b/source3/lib/ctdb_dummy.c
index 50de9ad..a3af030 100644
--- a/source3/lib/ctdb_dummy.c
+++ b/source3/lib/ctdb_dummy.c
@@ -30,13 +30,6 @@ NTSTATUS ctdbd_probe(void)
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-NTSTATUS ctdbd_messaging_send(struct ctdbd_connection *conn,
-			      uint32_t dst_vnn, uint64_t dst_srvid,
-			      struct messaging_rec *msg)
-{
-	return NT_STATUS_NOT_IMPLEMENTED;
-}
-
 NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
 				   uint32_t dst_vnn, uint64_t dst_srvid,
 				   const uint8_t *buf, size_t buflen)
diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index f730aaf..317603b 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -614,34 +614,6 @@ NTSTATUS ctdbd_register_msg_ctx(struct ctdbd_connection *conn,
 	return NT_STATUS_OK;
 }
 
-/*
- * Send a messaging message across a ctdbd
- */
-
-NTSTATUS ctdbd_messaging_send(struct ctdbd_connection *conn,
-			      uint32_t dst_vnn, uint64_t dst_srvid,
-			      struct messaging_rec *msg)
-{
-	DATA_BLOB blob;
-	NTSTATUS status;
-	enum ndr_err_code ndr_err;
-
-	ndr_err = ndr_push_struct_blob(
-		&blob, talloc_tos(), msg,
-		(ndr_push_flags_fn_t)ndr_push_messaging_rec);
-
-	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-		DEBUG(0, ("ndr_push_struct_blob failed: %s\n",
-			  ndr_errstr(ndr_err)));
-		return ndr_map_error2ntstatus(ndr_err);
-	}
-
-	status = ctdbd_messaging_send_blob(conn, dst_vnn, dst_srvid,
-					   blob.data, blob.length);
-	TALLOC_FREE(blob.data);
-	return status;
-}
-
 NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
 				   uint32_t dst_vnn, uint64_t dst_srvid,
 				   const uint8_t *buf, size_t buflen)
diff --git a/source3/lib/messages_ctdbd.c b/source3/lib/messages_ctdbd.c
index 430dc51..840b1ea 100644
--- a/source3/lib/messages_ctdbd.c
+++ b/source3/lib/messages_ctdbd.c
@@ -83,7 +83,9 @@ static int messaging_ctdb_send(struct server_id src,
 	struct messaging_rec msg;
 	uint8_t *buf;
 	ssize_t buflen;
+	DATA_BLOB blob;
 	NTSTATUS status;
+	enum ndr_err_code ndr_err;
 
 	if (num_fds > 0) {
 		return ENOSYS;
@@ -108,8 +110,19 @@ static int messaging_ctdb_send(struct server_id src,
 		.buf		= data_blob_const(buf, talloc_get_size(buf)),
 	};
 
-	status = ctdbd_messaging_send(ctx->conn, pid.vnn, pid.pid, &msg);
+	ndr_err = ndr_push_struct_blob(
+		&blob, buf, &msg,
+		(ndr_push_flags_fn_t)ndr_push_messaging_rec);
+
+	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+		DEBUG(0, ("ndr_push_struct_blob failed: %s\n",
+			  ndr_errstr(ndr_err)));
+		TALLOC_FREE(buf);
+		return ndr_map_error2errno(ndr_err);
+	}
 
+	status = ctdbd_messaging_send_blob(ctx->conn, pid.vnn, pid.pid,
+					   blob.data, blob.length);
 	TALLOC_FREE(buf);
 
 	if (NT_STATUS_IS_OK(status)) {
-- 
1.7.9.5


From 1128fc7e32759ab0177e7434627b0ac2abfb6012 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Wed, 20 May 2015 17:59:53 +0200
Subject: [PATCH 10/15] ctdbd_conn: ctdb_handle_message does not need msg_ctx

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/ctdbd_conn.c |    7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 317603b..3a9d094 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -521,8 +521,7 @@ int ctdbd_conn_get_fd(struct ctdbd_connection *conn)
 /*
  * Packet handler to receive and handle a ctdb message
  */
-static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
-				    struct ctdbd_connection *conn,
+static NTSTATUS ctdb_handle_message(struct ctdbd_connection *conn,
 				    struct ctdb_req_header *hdr)
 {
 	struct ctdb_req_message *msg;
@@ -555,8 +554,6 @@ static NTSTATUS ctdb_handle_message(struct messaging_context *msg_ctx,
 		return NT_STATUS_OK;
 	}
 
-	SMB_ASSERT(conn->msg_ctx != NULL);
-
 	ctdbd_msg_call_back(conn, msg);
 
 	return NT_STATUS_OK;
@@ -582,7 +579,7 @@ static void ctdbd_socket_handler(struct tevent_context *event_ctx,
 		cluster_fatal("ctdbd died\n");
 	}
 
-	status = ctdb_handle_message(conn->msg_ctx, conn, hdr);
+	status = ctdb_handle_message(conn, hdr);
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(10, ("could not handle incoming message: %s\n",
 			   nt_errstr(status)));
-- 
1.7.9.5


From 3414856abcaabb58e47455de311697ea48b34607 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Fri, 22 May 2015 18:40:25 +0200
Subject: [PATCH 11/15] messaging: With dgm_ref, don't destroy the dgm ctx

Since we use messaging_dgm_ref, we must rely on that to destroy the dgm
context when the last reference goes.

This is a real bugfix in case we have multiple messaging contexts.

I'm not sure if we should move towards just one single messaging context
per process, just like we have it for the dgm context.

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/messages.c |    2 --
 1 file changed, 2 deletions(-)

diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 51e88e2..6cc5275 100644
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -264,8 +264,6 @@ static int messaging_context_destructor(struct messaging_context *ctx)
 {
 	unsigned i;
 
-	messaging_dgm_destroy();
-
 	for (i=0; i<ctx->num_new_waiters; i++) {
 		if (ctx->new_waiters[i] != NULL) {
 			tevent_req_set_cleanup_fn(ctx->new_waiters[i], NULL);
-- 
1.7.9.5


From 1274bc1a7dfa714cb9a9386ae800352ccd44cef5 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Sun, 24 May 2015 21:25:56 +0200
Subject: [PATCH 12/15] messaging: Make messaging_dispatch_rec static

It's not needed in ctdbd_conn.c anymore

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/include/messages.h |    2 --
 source3/lib/messages.c     |    7 +++++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/source3/include/messages.h b/source3/include/messages.h
index 6b5e3da..c620f92 100644
--- a/source3/include/messages.h
+++ b/source3/include/messages.h
@@ -132,8 +132,6 @@ NTSTATUS messaging_send_iov(struct messaging_context *msg_ctx,
 			    struct server_id server, uint32_t msg_type,
 			    const struct iovec *iov, int iovlen,
 			    const int *fds, size_t num_fds);
-void messaging_dispatch_rec(struct messaging_context *msg_ctx,
-			    struct messaging_rec *rec);
 
 struct tevent_req *messaging_filtered_read_send(
 	TALLOC_CTX *mem_ctx, struct tevent_context *ev,
diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 6cc5275..26e5b01 100644
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -83,6 +83,9 @@ struct messaging_context {
 	struct server_id_db *names_db;
 };
 
+static void messaging_dispatch_rec(struct messaging_context *msg_ctx,
+				   struct messaging_rec *rec);
+
 /****************************************************************************
  A useful function for testing the message system.
 ****************************************************************************/
@@ -923,8 +926,8 @@ static bool messaging_append_new_waiters(struct messaging_context *msg_ctx)
 /*
   Dispatch one messaging_rec
 */
-void messaging_dispatch_rec(struct messaging_context *msg_ctx,
-			    struct messaging_rec *rec)
+static void messaging_dispatch_rec(struct messaging_context *msg_ctx,
+				   struct messaging_rec *rec)
 {
 	struct messaging_callback *cb, *next;
 	unsigned i;
-- 
1.7.9.5


From 71b6cf9c2aa74af05ca46d88341eb20a3b68f920 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Mon, 25 May 2015 08:50:35 +0200
Subject: [PATCH 13/15] ctdbd_conn: Introduce ctdbd_messaging_send_iov()

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/include/ctdbd_conn.h   |    6 +++---
 source3/lib/ctdb_dummy.c       |    6 +++---
 source3/lib/ctdbd_conn.c       |   19 ++++++++++---------
 source3/lib/messages_ctdbd.c   |    8 ++++++--
 source3/smbd/notify_internal.c |   10 +++++++---
 5 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/source3/include/ctdbd_conn.h b/source3/include/ctdbd_conn.h
index 6b3ebf1..a404724 100644
--- a/source3/include/ctdbd_conn.h
+++ b/source3/include/ctdbd_conn.h
@@ -38,9 +38,9 @@ struct messaging_context *ctdb_conn_msg_ctx(struct ctdbd_connection *conn);
 
 int ctdbd_conn_get_fd(struct ctdbd_connection *conn);
 
-NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
-				   uint32_t dst_vnn, uint64_t dst_srvid,
-				   const uint8_t *buf, size_t buflen);
+NTSTATUS ctdbd_messaging_send_iov(struct ctdbd_connection *conn,
+				  uint32_t dst_vnn, uint64_t dst_srvid,
+				  const struct iovec *iov, int iovlen);
 
 bool ctdbd_process_exists(struct ctdbd_connection *conn, uint32_t vnn,
 			  pid_t pid);
diff --git a/source3/lib/ctdb_dummy.c b/source3/lib/ctdb_dummy.c
index a3af030..bea707d 100644
--- a/source3/lib/ctdb_dummy.c
+++ b/source3/lib/ctdb_dummy.c
@@ -30,9 +30,9 @@ NTSTATUS ctdbd_probe(void)
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
 
-NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
-				   uint32_t dst_vnn, uint64_t dst_srvid,
-				   const uint8_t *buf, size_t buflen)
+NTSTATUS ctdbd_messaging_send_iov(struct ctdbd_connection *conn,
+				  uint32_t dst_vnn, uint64_t dst_srvid,
+				  const struct iovec *iov, int iovlen)
 {
 	return NT_STATUS_NOT_IMPLEMENTED;
 }
diff --git a/source3/lib/ctdbd_conn.c b/source3/lib/ctdbd_conn.c
index 3a9d094..1838e97 100644
--- a/source3/lib/ctdbd_conn.c
+++ b/source3/lib/ctdbd_conn.c
@@ -24,6 +24,7 @@
 #include "ctdbd_conn.h"
 #include "system/select.h"
 #include "lib/sys_rw_data.h"
+#include "lib/util/iov_buf.h"
 
 #include "messages.h"
 
@@ -611,12 +612,13 @@ NTSTATUS ctdbd_register_msg_ctx(struct ctdbd_connection *conn,
 	return NT_STATUS_OK;
 }
 
-NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
-				   uint32_t dst_vnn, uint64_t dst_srvid,
-				   const uint8_t *buf, size_t buflen)
+NTSTATUS ctdbd_messaging_send_iov(struct ctdbd_connection *conn,
+				  uint32_t dst_vnn, uint64_t dst_srvid,
+				  const struct iovec *iov, int iovlen)
 {
 	struct ctdb_req_message r;
-	struct iovec iov[2];
+	struct iovec iov2[iovlen+1];
+	size_t buflen = iov_buflen(iov, iovlen);
 	ssize_t nwritten;
 
 	r.hdr.length = offsetof(struct ctdb_req_message, data) + buflen;
@@ -633,12 +635,11 @@ NTSTATUS ctdbd_messaging_send_blob(struct ctdbd_connection *conn,
 	DEBUG(10, ("ctdbd_messaging_send: Sending ctdb packet\n"));
 	ctdb_packet_dump(&r.hdr);
 
-	iov[0].iov_base = &r;
-	iov[0].iov_len = offsetof(struct ctdb_req_message, data);
-	iov[1].iov_base = discard_const_p(uint8_t, buf);
-	iov[1].iov_len = buflen;
+	iov2[0].iov_base = &r;
+	iov2[0].iov_len = offsetof(struct ctdb_req_message, data);
+	memcpy(&iov2[1], iov, iovlen * sizeof(struct iovec));
 
-	nwritten = write_data_iov(conn->fd, iov, ARRAY_SIZE(iov));
+	nwritten = write_data_iov(conn->fd, iov2, iovlen+1);
 	if (nwritten == -1) {
 		DEBUG(3, ("write_data_iov failed: %s\n", strerror(errno)));
 		cluster_fatal("cluster dispatch daemon msg write error\n");
diff --git a/source3/lib/messages_ctdbd.c b/source3/lib/messages_ctdbd.c
index 840b1ea..bf9c89f 100644
--- a/source3/lib/messages_ctdbd.c
+++ b/source3/lib/messages_ctdbd.c
@@ -84,6 +84,7 @@ static int messaging_ctdb_send(struct server_id src,
 	uint8_t *buf;
 	ssize_t buflen;
 	DATA_BLOB blob;
+	struct iovec iov2;
 	NTSTATUS status;
 	enum ndr_err_code ndr_err;
 
@@ -121,8 +122,11 @@ static int messaging_ctdb_send(struct server_id src,
 		return ndr_map_error2errno(ndr_err);
 	}
 
-	status = ctdbd_messaging_send_blob(ctx->conn, pid.vnn, pid.pid,
-					   blob.data, blob.length);
+	iov2 = (struct iovec) { .iov_base = blob.data,
+				.iov_len = blob.length };
+
+	status = ctdbd_messaging_send_iov(ctx->conn, pid.vnn, pid.pid,
+					  &iov2, 1);
 	TALLOC_FREE(buf);
 
 	if (NT_STATUS_IS_OK(status)) {
diff --git a/source3/smbd/notify_internal.c b/source3/smbd/notify_internal.c
index 845a7cc..dc50d4f 100644
--- a/source3/smbd/notify_internal.c
+++ b/source3/smbd/notify_internal.c
@@ -627,6 +627,7 @@ void notify_trigger(struct notify_context *notify,
 	uint32_t last_vnn;
 	uint8_t *remote_blob = NULL;
 	size_t remote_blob_len = 0;
+	struct iovec iov;
 	char *path, *to_free;
 	char tmpbuf[PATH_MAX];
 	ssize_t len;
@@ -698,6 +699,9 @@ void notify_trigger(struct notify_context *notify,
 		goto done;
 	}
 
+	iov = (struct iovec) { .iov_base = remote_blob,
+			       .iov_len = remote_blob_len };
+
 	for (i=0; i<num_vnns; i++) {
 		uint32_t vnn = idx_state.vnns[i];
 		NTSTATUS status;
@@ -706,11 +710,11 @@ void notify_trigger(struct notify_context *notify,
 			continue;
 		}
 
-		status = ctdbd_messaging_send_blob(
+		status = ctdbd_messaging_send_iov(
 			ctdbd_conn, vnn, CTDB_SRVID_SAMBA_NOTIFY_PROXY,
-			remote_blob, remote_blob_len);
+			&iov, 1);
 		if (!NT_STATUS_IS_OK(status)) {
-			DEBUG(10, ("ctdbd_messaging_send_blob to vnn %d "
+			DEBUG(10, ("ctdbd_messaging_send_iov to vnn %d "
 				   "returned %s, ignoring\n", (int)vnn,
 				   nt_errstr(status)));
 		}
-- 
1.7.9.5


From 6b62d5591699d96ac4f4e9313fdc4a9c628db919 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Mon, 25 May 2015 17:50:46 +0000
Subject: [PATCH 14/15] messaging: Remove an unused variable

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/messages.c |    7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 26e5b01..0297cc8 100644
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -213,13 +213,12 @@ static void messaging_recv_cb(const uint8_t *msg, size_t msg_len,
 {
 	struct messaging_context *msg_ctx = talloc_get_type_abort(
 		private_data, struct messaging_context);
-	uint8_t hdr[MESSAGE_HDR_LENGTH];
 	struct server_id_buf idbuf;
 	struct messaging_rec rec;
 	int64_t fds64[MIN(num_fds, INT8_MAX)];
 	size_t i;
 
-	if (msg_len < sizeof(hdr)) {
+	if (msg_len < MESSAGE_HDR_LENGTH) {
 		for (i=0; i < num_fds; i++) {
 			close(fds[i]);
 		}
@@ -246,8 +245,8 @@ static void messaging_recv_cb(const uint8_t *msg, size_t msg_len,
 
 	rec = (struct messaging_rec) {
 		.msg_version = MESSAGE_VERSION,
-		.buf.data = discard_const_p(uint8_t, msg) + sizeof(hdr),
-		.buf.length = msg_len - sizeof(hdr),
+		.buf.data = discard_const_p(uint8_t, msg) + MESSAGE_HDR_LENGTH,
+		.buf.length = msg_len - MESSAGE_HDR_LENGTH,
 		.num_fds = num_fds,
 		.fds = fds64,
 	};
-- 
1.7.9.5


From 0cc574a8e370ca1235022fab125f23c3236f5965 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Mon, 25 May 2015 17:59:22 +0000
Subject: [PATCH 15/15] messages_ctdb: Use message_hdr_[get/put]

This also avoids the message copy when sending to ctdb by
using ctdbd_messaging_send_iov

Signed-off-by: Volker Lendecke <vl at samba.org>
---
 source3/lib/messages_ctdbd.c |  143 ++++++++++++++----------------------------
 1 file changed, 47 insertions(+), 96 deletions(-)

diff --git a/source3/lib/messages_ctdbd.c b/source3/lib/messages_ctdbd.c
index bf9c89f..0ce4da7 100644
--- a/source3/lib/messages_ctdbd.c
+++ b/source3/lib/messages_ctdbd.c
@@ -21,6 +21,7 @@
 #include "messages.h"
 #include "util_tdb.h"
 #include "lib/util/iov_buf.h"
+#include "lib/messages_util.h"
 
 #include "ctdb.h"
 #include "ctdb_private.h"
@@ -80,55 +81,20 @@ static int messaging_ctdb_send(struct server_id src,
 {
 	struct messaging_ctdbd_context *ctx = talloc_get_type_abort(
 		backend->private_data, struct messaging_ctdbd_context);
-	struct messaging_rec msg;
-	uint8_t *buf;
-	ssize_t buflen;
-	DATA_BLOB blob;
-	struct iovec iov2;
+	uint8_t hdr[MESSAGE_HDR_LENGTH];
+	struct iovec iov2[iovlen+1];
 	NTSTATUS status;
-	enum ndr_err_code ndr_err;
 
 	if (num_fds > 0) {
 		return ENOSYS;
 	}
 
-	buflen = iov_buflen(iov, iovlen);
-	if (buflen == -1) {
-		return EMSGSIZE;
-	}
-
-	buf = talloc_array(talloc_tos(), uint8_t, buflen);
-	if (buf == NULL) {
-		return ENOMEM;
-	}
-	iov_buf(iov, iovlen, buf, buflen);
-
-	msg = (struct messaging_rec) {
-		.msg_version	= MESSAGE_VERSION,
-		.msg_type	= msg_type,
-		.dest		= pid,
-		.src		= src,
-		.buf		= data_blob_const(buf, talloc_get_size(buf)),
-	};
-
-	ndr_err = ndr_push_struct_blob(
-		&blob, buf, &msg,
-		(ndr_push_flags_fn_t)ndr_push_messaging_rec);
-
-	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-		DEBUG(0, ("ndr_push_struct_blob failed: %s\n",
-			  ndr_errstr(ndr_err)));
-		TALLOC_FREE(buf);
-		return ndr_map_error2errno(ndr_err);
-	}
-
-	iov2 = (struct iovec) { .iov_base = blob.data,
-				.iov_len = blob.length };
+	message_hdr_put(hdr, msg_type, src, pid);
+	iov2[0] = (struct iovec){ .iov_base = hdr, .iov_len = sizeof(hdr) };
+	memcpy(&iov2[1], iov, iovlen * sizeof(*iov));
 
 	status = ctdbd_messaging_send_iov(ctx->conn, pid.vnn, pid.pid,
-					  &iov2, 1);
-	TALLOC_FREE(buf);
-
+					  iov2, iovlen+1);
 	if (NT_STATUS_IS_OK(status)) {
 		return 0;
 	}
@@ -145,81 +111,66 @@ static int messaging_ctdbd_destructor(struct messaging_ctdbd_context *ctx)
 	return 0;
 }
 
-static struct messaging_rec *ctdb_pull_messaging_rec(
-	TALLOC_CTX *mem_ctx, const struct ctdb_req_message *msg)
-{
-	struct messaging_rec *result;
-	DATA_BLOB blob;
-	enum ndr_err_code ndr_err;
-	size_t len = msg->hdr.length;
-
-	if (len < offsetof(struct ctdb_req_message, data)) {
-		return NULL;
-	}
-	len -= offsetof(struct ctdb_req_message, data);
-
-	if (len < msg->datalen) {
-		return NULL;
-	}
-
-	result = talloc(mem_ctx, struct messaging_rec);
-	if (result == NULL) {
-		return NULL;
-	}
-
-	blob = data_blob_const(msg->data, msg->datalen);
-
-	ndr_err = ndr_pull_struct_blob_all(
-		&blob, result, result,
-		(ndr_pull_flags_fn_t)ndr_pull_messaging_rec);
-
-	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-		DEBUG(0, ("ndr_pull_struct_blob failed: %s\n",
-			  ndr_errstr(ndr_err)));
-		TALLOC_FREE(result);
-		return NULL;
-	}
-
-	if (DEBUGLEVEL >= 11) {
-		DEBUG(11, ("ctdb_pull_messaging_rec:\n"));
-		NDR_PRINT_DEBUG(messaging_rec, result);
-	}
-
-	return result;
-}
-
 static void messaging_ctdb_recv(struct ctdb_req_message *msg,
 				void *private_data)
 {
 	struct messaging_context *msg_ctx = talloc_get_type_abort(
 		private_data, struct messaging_context);
 	struct server_id me = messaging_server_id(msg_ctx);
-	struct messaging_rec *rec;
 	NTSTATUS status;
 	struct iovec iov;
+	size_t msg_len;
+	uint8_t *msg_buf;
+	struct server_id src, dst;
+	enum messaging_type msg_type;
+	struct server_id_buf idbuf;
+
+	msg_len = msg->hdr.length;
+	if (msg_len < offsetof(struct ctdb_req_message, data)) {
+		DEBUG(10, ("%s: len %u too small\n", __func__,
+			   (unsigned)msg_len));
+		return;
+	}
+	msg_len -= offsetof(struct ctdb_req_message, data);
 
-	rec = ctdb_pull_messaging_rec(msg_ctx, msg);
-	if (rec == NULL) {
-		DEBUG(10, ("%s: ctdb_pull_messaging_rec failed\n", __func__));
+	if (msg_len < msg->datalen) {
+		DEBUG(10, ("%s: msg_len=%u < msg->datalen=%u\n", __func__,
+			   (unsigned)msg_len, (unsigned)msg->datalen));
 		return;
 	}
 
-	if (!server_id_same_process(&me, &rec->dest)) {
+	if (msg_len < MESSAGE_HDR_LENGTH) {
+		DEBUG(1, ("%s: message too short: %u\n", __func__,
+			  (unsigned)msg_len));
+		return;
+	}
+
+	message_hdr_get(&msg_type, &src, &dst, msg->data);
+
+	msg_len -= MESSAGE_HDR_LENGTH;
+	msg_buf = msg->data + MESSAGE_HDR_LENGTH;
+
+	DEBUG(10, ("%s: Received message 0x%x len %u from %s\n",
+		   __func__, (unsigned)msg_type, (unsigned)msg_len,
+		   server_id_str_buf(src, &idbuf)));
+
+	if (!server_id_same_process(&me, &dst)) {
 		struct server_id_buf id1, id2;
 
 		DEBUG(10, ("%s: I'm %s, ignoring msg to %s\n", __func__,
 			   server_id_str_buf(me, &id1),
-			   server_id_str_buf(rec->dest, &id2)));
-		TALLOC_FREE(rec);
+			   server_id_str_buf(dst, &id2)));
 		return;
 	}
 
-	iov = (struct iovec) { .iov_base = rec->buf.data,
-			       .iov_len = rec->buf.length };
+	iov = (struct iovec) { .iov_base = msg_buf, .iov_len = msg_len };
+
+	/*
+	 * Go through the event loop
+	 */
 
-	status = messaging_send_iov_from(msg_ctx, rec->src, rec->dest,
-					 rec->msg_type, &iov, 1, NULL, 0);
-	TALLOC_FREE(rec);
+	status = messaging_send_iov_from(msg_ctx, src, dst, msg_type,
+					 &iov, 1, NULL, 0);
 
 	if (!NT_STATUS_IS_OK(status)) {
 		DEBUG(10, ("%s: messaging_send_iov_from failed: %s\n",
-- 
1.7.9.5



More information about the samba-technical mailing list