Rev 614: new approach for killing TCP connections on IP release in http://samba.org/~tridge/ctdb

tridge at samba.org tridge at samba.org
Thu Sep 13 00:24:49 GMT 2007


------------------------------------------------------------
revno: 614
revision-id: tridge at samba.org-20070913002448-dh97gw7kfv4iavem
parent: tridge at samba.org-20070913000318-6il4eeaj25ibvoth
committer: Andrew Tridgell <tridge at samba.org>
branch nick: tridge.test2
timestamp: Thu 2007-09-13 10:24:48 +1000
message:
  new approach for killing TCP connections on IP release
modified:
  config/events.d/10.interface   10.interface-20070604050809-s21zslfirn07zjt8-1
  config/events.d/60.nfs         nfs-20070601141008-hy3h4qgbk1jd2jci-1
  tools/ctdb_diagnostics         ctdb_diagnostics-20070905041904-9d9r1qnt1j9qiwiz-1
=== modified file 'config/events.d/10.interface'
--- a/config/events.d/10.interface	2007-09-12 03:23:36 +0000
+++ b/config/events.d/10.interface	2007-09-13 00:24:48 +0000
@@ -20,6 +20,37 @@
 	exit 0
 }
 
+################################################
+# kill off any TCP connections with the given IP
+kill_tcp_connections() {
+    _IP="$1"    
+    _failed=0
+    _killcount=0
+    netstat -tn |egrep "^tcp.*\s+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' | 
+    while read dest src; do
+	srcip=`echo $src | cut -d: -f1`
+	srcport=`echo $src | cut -d: -f2`
+	destip=`echo $dest | cut -d: -f1`
+	destport=`echo $dest | cut -d: -f2`
+	ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
+	_killcount=`expr $_killcount + 1`
+    done
+    [ _failed = 0 ] || {
+	echo "Failed to send killtcp control"
+	return;
+    }
+    _count=0
+    while netstat -tn | grep $_IP > /dev/null; do
+	sleep 1
+	_count=`expr $_count + 1`
+	[ $_count -gt 5 ] && {
+	    echo "Timed out killing tcp connections for IP $_IP"
+	    return;
+	}
+    done
+    echo "`date` killed $_killcount TCP connections to released IP $_IP"
+}
+
 case $cmd in 
      #############################
      # called when ctdbd starts up
@@ -29,11 +60,6 @@
 	[ -f /proc/sys/net/ipv4/conf/all/arp_filter ] && {
 	    echo 1 > /proc/sys/net/ipv4/conf/all/arp_filter
 	}
-	# make sure we don't reply to arp requests for IPs we have moved to scope
-	# host on loopback
-	[ -f /proc/sys/net/ipv4/conf/all/arp_ignore ] && {
-	    echo 3 > /proc/sys/net/ipv4/conf/all/arp_ignore
-	}
 	;;
 
 
@@ -53,11 +79,12 @@
 		 echo "`/bin/date` Failed to bringup interface $iface"
 		 exit 1
 	}
-	/sbin/ip addr del $ip/32 dev lo >/dev/null 2>/dev/null
 	/sbin/ip addr add $ip/$maskbits dev $iface || {
 		 echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
 		 exit 1
 	}
+	# cope with the script being killed while we have the interface blocked
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 
 	# flush our route cache
 	echo 1 > /proc/sys/net/ipv4/route/flush
@@ -71,16 +98,32 @@
 	   echo "`/bin/date` must supply interface, IP and maskbits"
 	   exit 1
 	fi
+
+	# releasing an IP is a bit more complex than it seems. Once the IP
+	# is released, any open tcp connections to that IP on this host will end
+	# up being stuck. Some of them (such as NFS connections) will be unkillable
+	# so we need to use the killtcp ctdb function to kill them off. We also
+	# need to make sure that no new connections get established while we are 
+	# doing this! So what we do is this:
+	# 1) firewall this IP, so no new external packets arrive for it
+	# 2) use netstat -tn to find existing connections, and kill them 
+	# 3) remove the IP from the interface
+	# 4) remove the firewall rule
 	iface=$1
 	ip=$2
 	maskbits=$3
-	/sbin/ip addr del $ip/$maskbits dev $iface || {
+
+	failed=0
+	# we do an extra delete to cope with the script being killed
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
+	/sbin/iptables -I INPUT -i $iface -d $ip -j DROP
+	kill_tcp_connections $ip
+	/sbin/ip addr del $ip/$maskbits dev $iface || failed=1
+	/sbin/iptables -D INPUT -i $iface -d $ip -j DROP
+	[ $failed = 0 ] || {
 		 echo "`/bin/date` Failed to del $ip on dev $iface"
 		 exit 1
 	}
-	# we put the IP on loopback so our killtcp code can work, this allows
-	# us to avoid restarting the NFS server when we release an IP
-	/sbin/ip addr add $ip/32 dev lo scope host >/dev/null 2>/dev/null
 
 	# flush our route cache
 	echo 1 > /proc/sys/net/ipv4/route/flush

=== modified file 'config/events.d/60.nfs'
--- a/config/events.d/60.nfs	2007-09-09 21:45:57 +0000
+++ b/config/events.d/60.nfs	2007-09-13 00:24:48 +0000
@@ -55,37 +55,7 @@
 	maskbits=$3
 
 	echo $ip >> /etc/ctdb/state/statd/restart
-
 	/bin/rm -f /etc/ctdb/state/statd/ip/$ip
-
-	# RST all tcp connections to the lockmanager
-	[ ! -z "$LOCKD_TCPPORT" ] && {
-	        # RST all tcp connections used for NLM to ensure that they do
-		# not survive in ESTABLISHED state across a failover/failback
-		# and create an ack storm
-		netstat -tn |egrep "^tcp.*\s+$ip:${LOCKD_TCPPORT}\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-			srcip=`echo $src | cut -d: -f1`
-			srcport=`echo $src | cut -d: -f2`
-			destip=`echo $dest | cut -d: -f1`
-			destport=`echo $dest | cut -d: -f2`
-			ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#			ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-		done
-	} > /dev/null 2>&1
-
-
-        # RST the local side for all tcp connections used for NFS to ensure 
-	# that they do not survive in ESTABLISHED state across a 
-	# failover/failback and create an ack storm
-	netstat -tn |egrep "^tcp.*\s+$ip:2049\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
-		srcip=`echo $src | cut -d: -f1`
-		srcport=`echo $src | cut -d: -f2`
-		destip=`echo $dest | cut -d: -f1`
-		destport=`echo $dest | cut -d: -f2`
-		ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 
-#		ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
-	done
-
 	exit 0
 	;;
 

=== modified file 'tools/ctdb_diagnostics'
--- a/tools/ctdb_diagnostics	2007-09-10 01:27:07 +0000
+++ b/tools/ctdb_diagnostics	2007-09-13 00:24:48 +0000
@@ -123,6 +123,7 @@
 show_all "/sbin/route -n"
 show_all "crontab -l"
 show_all "sysctl -a"
+show_all "/sbin/iptables -L -n"
 
 [ -d /usr/lpp/mmfs ] && {
 cat <<EOF



More information about the samba-cvs mailing list