Rev 50: bugfix in ibw_send in http://samba.org/~tridge/psomogyi/

[psomogyi at gamax.hu]

------------------------------------------------------------
revno: 50
revision-id: psomogyi at gamax.hu-20070104154441-ekn574yyvselq7e3
parent: psomogyi at gamax.hu-20070103163747-9hm63hrzkvpkkpxa
committer: Peter Somogyi <psomogyi at gamax.hu>
branch nick: ctdb
timestamp: Thu 2007-01-04 16:44:41 +0100
message:
  bugfix in ibw_send
  Forgot to allow different message pointer than beginning of an allocated buf.
modified:
  ib/ibw_ctdb_init.c             ibw_ctdb_init.c-20070102171305-cn2z4k7ibx8141d5-1
  ib/ibwrapper.c                 ibwrapper.c-20061204130028-0125b4f5a72f4b11
  ib/ibwrapper_internal.h        ibwrapper_internal.h-20061204130028-47f0a7e658b16ca2
=== modified file 'ib/ibw_ctdb_init.c'
--- a/ib/ibw_ctdb_init.c	2007-01-02 17:16:39 +0000
+++ b/ib/ibw_ctdb_init.c	2007-01-04 15:44:41 +0000
@@ -135,7 +135,9 @@
 	.add_node  = ctdb_ibw_add_node,
 	.queue_pkt = ctdb_ibw_queue_pkt,
 	.allocate_pkt = ctdb_ibw_allocate_pkt
-	/* TODO: missing node_disconnect & final_stop upcalls */
+	
+//	.dealloc_pkt = ctdb_ibw_dealloc_pkt
+//	.stop = ctdb_ibw_stop
 };
 
 /*

=== modified file 'ib/ibwrapper.c'
--- a/ib/ibwrapper.c	2007-01-03 16:37:47 +0000
+++ b/ib/ibwrapper.c	2007-01-04 15:44:41 +0000
@@ -628,14 +628,13 @@
 	}
 
 	if (pconn->queue) {
-		char	*buf;
-
-		DEBUG(10, ("ibw_wc_send#queue %u", (int)wc->wr_id));
+		DEBUG(10, ("ibw_wc_send#queue %u\n", (int)wc->wr_id));
 		p = pconn->queue;
 		DLIST_REMOVE(pconn->queue, p);
 
-		buf = (p->msg_large!=NULL) ? p->msg_large : p->msg;
-		ibw_send(conn, buf, p, ntohl(*(uint32_t *)buf));
+		assert(p->queued_msg!=NULL);
+		ibw_send(conn, p->queued_msg, p, ntohl(*(uint32_t *)p->queued_msg));
+		p->queued_msg = NULL;
 	}
 
 	return 0;
@@ -1033,15 +1032,15 @@
 		DLIST_REMOVE(pconn->wr_list_avail, p);
 		DLIST_ADD(pconn->wr_list_used, p);
 
-		if (len + sizeof(uint32_t) <= pctx->opts.avg_send_size) {
-			*buf = (void *)(p->msg + sizeof(uint32_t));
+		if (len <= pctx->opts.avg_send_size) {
+			*buf = (void *)p->msg;
 		} else {
-			p->msg_large = ibw_alloc_mr(pctx, pconn, len + sizeof(uint32_t), &p->mr_large);
+			p->msg_large = ibw_alloc_mr(pctx, pconn, len, &p->mr_large);
 			if (!p->msg_large) {
 				sprintf(ibw_lasterr, "ibw_alloc_mr#1 failed\n");
 				goto error;
 			}
-			*buf = (void *)(p->msg_large + sizeof(uint32_t));
+			*buf = (void *)p->msg_large;
 		}
 	} else {
 		DEBUG(10, ("ibw_alloc_send_buf#2: cmid=%u, len=%d\n", (uint32_t)pconn->cm_id, len));
@@ -1065,12 +1064,12 @@
 		}
 		DLIST_REMOVE(pconn->extra_avail, p);
 
-		p->msg_large = ibw_alloc_mr(pctx, pconn, len + sizeof(uint32_t), &p->mr_large);
+		p->msg_large = ibw_alloc_mr(pctx, pconn, len, &p->mr_large);
 		if (!p->msg_large) {
 			sprintf(ibw_lasterr, "ibw_alloc_mr#2 failed");
 			goto error;
 		}
-		*buf = (void *)(p->msg_large + sizeof(uint32_t));
+		*buf = (void *)p->msg_large;
 	}
 
 	*key = (void *)p;
@@ -1110,13 +1109,12 @@
 		DEBUG(10, ("ibw_wc_send#1(cmid: %u, wrid: %u, n: %d)\n",
 			(uint32_t)pconn->cm_id, (uint32_t)wr.wr_id, len));
 
+		list.addr = (uintptr_t)buf;
 		if (p->msg_large==NULL) {
 			list.lkey = pconn->mr_send->lkey;
-			list.addr = (uintptr_t) p->msg;
 		} else {
 			assert(p->mr_large!=NULL);
 			list.lkey = p->mr_large->lkey;
-			list.addr = (uintptr_t) p->msg_large;
 		}
 
 		rc = ibv_post_send(pconn->cm_id->qp, &wr, &bad_wr);
@@ -1140,6 +1138,7 @@
 
 	/* to be sent by ibw_wc_send */
 	DLIST_ADD_END(pconn->queue, p, struct ibw_wr *); /* TODO: optimize */
+	p->queued_msg = buf;
 
 	return 0;
 }

=== modified file 'ib/ibwrapper_internal.h'
--- a/ib/ibwrapper_internal.h	2006-12-21 16:41:48 +0000
+++ b/ib/ibwrapper_internal.h	2007-01-04 15:44:41 +0000
@@ -36,6 +36,8 @@
 	char	*msg_large; /* allocated specially for "large" message */
 	struct ibv_mr *mr_large;
 
+	char	*queued_msg; /* set at ibw_send - can be different than above */
+
 	struct ibw_wr *next, *prev; /* in wr_list_avail or wr_list_used */
 };