Rev 667: fixed a double close of a socket, leading to an EPOLL error in http://samba.org/~tridge/ctdb

[tridge at samba.org]

------------------------------------------------------------
revno: 667
revision-id: tridge at samba.org-20071022064111-v3s7tduciucdefbb
parent: tridge at samba.org-20071022051332-08d6bfu5donk4cg5
committer: Andrew Tridgell <tridge at samba.org>
branch nick: tridge
timestamp: Mon 2007-10-22 16:41:11 +1000
message:
  fixed a double close of a socket, leading to an EPOLL error
modified:
  tcp/ctdb_tcp.h                 ctdb_tcp.h-20061127103747-l8xeniwiapbydehq-3
  tcp/tcp_connect.c              tcp_connect.c-20061128004937-x70q1cu5xzg5g2tm-1
  tcp/tcp_init.c                 tcp_init.c-20061128004937-x70q1cu5xzg5g2tm-2
=== modified file 'tcp/ctdb_tcp.h'
--- a/tcp/ctdb_tcp.h	2007-07-10 05:29:31 +0000
+++ b/tcp/ctdb_tcp.h	2007-10-22 06:41:11 +0000
@@ -50,5 +50,6 @@
 			   struct timeval t, void *private_data);
 void ctdb_tcp_read_cb(uint8_t *data, size_t cnt, void *args);
 void ctdb_tcp_tnode_cb(uint8_t *data, size_t cnt, void *private_data);
+void ctdb_tcp_stop_connection(struct ctdb_node *node);
 
 #define CTDB_TCP_ALIGNMENT 8

=== modified file 'tcp/tcp_connect.c'
--- a/tcp/tcp_connect.c	2007-10-22 05:13:32 +0000
+++ b/tcp/tcp_connect.c	2007-10-22 06:41:11 +0000
@@ -26,6 +26,26 @@
 #include "ctdb_tcp.h"
 
 /*
+  stop any connecting (established or pending) to a node
+ */
+void ctdb_tcp_stop_connection(struct ctdb_node *node)
+{
+	struct ctdb_tcp_node *tnode = talloc_get_type(
+		node->private_data, struct ctdb_tcp_node);
+	
+	ctdb_queue_set_fd(tnode->out_queue, -1);
+	talloc_free(tnode->connect_te);
+	talloc_free(tnode->connect_fde);
+	tnode->connect_fde = NULL;
+	tnode->connect_te = NULL;
+	if (tnode->fd != -1) {
+		close(tnode->fd);
+		tnode->fd = -1;
+	}
+}
+
+
+/*
   called when a complete packet has come in - should not happen on this socket
  */
 void ctdb_tcp_tnode_cb(uint8_t *data, size_t cnt, void *private_data)
@@ -38,12 +58,9 @@
 		node->ctdb->upcalls->node_dead(node);
 	}
 
-	/* start a new connect cycle to try to re-establish the
-	   link */
-	ctdb_queue_set_fd(tnode->out_queue, -1);
-	tnode->fd = -1;
-	event_add_timed(node->ctdb->ev, tnode, timeval_zero(), 
-			ctdb_tcp_node_connect, node);
+	ctdb_tcp_stop_connection(node);
+	tnode->connect_te = event_add_timed(node->ctdb->ev, tnode, timeval_zero(),
+					    ctdb_tcp_node_connect, node);
 }
 
 /*
@@ -66,23 +83,24 @@
 
 	if (getsockopt(tnode->fd, SOL_SOCKET, SO_ERROR, &error, &len) != 0 ||
 	    error != 0) {
-		talloc_free(tnode->connect_fde);
-		tnode->connect_fde = NULL;
-		close(tnode->fd);
-		tnode->fd = -1;
-		event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), 
-				ctdb_tcp_node_connect, node);
+		ctdb_tcp_stop_connection(node);
+		tnode->connect_te = event_add_timed(ctdb->ev, tnode, 
+						    timeval_current_ofs(1, 0),
+						    ctdb_tcp_node_connect, node);
 		return;
 	}
 
 	talloc_free(tnode->connect_fde);
 	tnode->connect_fde = NULL;
-	
+
         setsockopt(tnode->fd,IPPROTO_TCP,TCP_NODELAY,(char *)&one,sizeof(one));
         setsockopt(tnode->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one));
 
 	ctdb_queue_set_fd(tnode->out_queue, tnode->fd);
 
+	/* the queue subsystem now owns this fd */
+	tnode->fd = -1;
+       
 	/* tell the ctdb layer we are connected */
 	node->ctdb->upcalls->node_connected(node);
 }
@@ -117,12 +135,7 @@
         struct sockaddr_in sock_in;
         struct sockaddr_in sock_out;
 
-	if (tnode->fd != -1) {
-		talloc_free(tnode->connect_fde);
-		tnode->connect_fde = NULL;
-		close(tnode->fd);
-		tnode->fd = -1;
-	}
+	ctdb_tcp_stop_connection(node);
 
 	tnode->fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 
@@ -159,11 +172,10 @@
 
 	if (connect(tnode->fd, (struct sockaddr *)&sock_out, sizeof(sock_out)) != 0 &&
 	    errno != EINPROGRESS) {
-		/* try again once a second */
-		close(tnode->fd);
-		tnode->fd = -1;
-		event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), 
-				ctdb_tcp_node_connect, node);
+		ctdb_tcp_stop_connection(node);
+		tnode->connect_te = event_add_timed(ctdb->ev, tnode, 
+						    timeval_current_ofs(1, 0),
+						    ctdb_tcp_node_connect, node);
 		return;
 	}
 

=== modified file 'tcp/tcp_init.c'
--- a/tcp/tcp_init.c	2007-10-19 05:19:25 +0000
+++ b/tcp/tcp_init.c	2007-10-22 06:41:11 +0000
@@ -80,8 +80,9 @@
 		struct ctdb_tcp_node *tnode = talloc_get_type(
 			node->private_data, struct ctdb_tcp_node);
 		if (!ctdb_same_address(&ctdb->address, &node->address)) {
-			event_add_timed(ctdb->ev, tnode, timeval_zero(), 
-					ctdb_tcp_node_connect, node);
+			tnode->connect_te = event_add_timed(ctdb->ev, tnode, 
+							    timeval_zero(), 
+							    ctdb_tcp_node_connect, node);
 		}
 	}
 
@@ -99,11 +100,10 @@
 
 	DEBUG(0,("Tearing down connection to dead node :%d\n", node->pnn));
 
-	tnode->fd = -1;
-	ctdb_queue_set_fd(tnode->out_queue, -1);
+	ctdb_tcp_stop_connection(node);
 
-	event_add_timed(node->ctdb->ev, tnode, timeval_zero(), 
-			ctdb_tcp_node_connect, node);
+	tnode->connect_te = event_add_timed(node->ctdb->ev, tnode, timeval_zero(), 
+					    ctdb_tcp_node_connect, node);
 }