Rev 614: new approach for killing TCP connections on IP release in
http://samba.org/~tridge/ctdb
tridge at samba.org
tridge at samba.org
Thu Sep 13 00:24:49 GMT 2007
------------------------------------------------------------
revno: 614
revision-id: tridge at samba.org-20070913002448-dh97gw7kfv4iavem
parent: tridge at samba.org-20070913000318-6il4eeaj25ibvoth
committer: Andrew Tridgell <tridge at samba.org>
branch nick: tridge.test2
timestamp: Thu 2007-09-13 10:24:48 +1000
message:
new approach for killing TCP connections on IP release
modified:
config/events.d/10.interface 10.interface-20070604050809-s21zslfirn07zjt8-1
config/events.d/60.nfs nfs-20070601141008-hy3h4qgbk1jd2jci-1
tools/ctdb_diagnostics ctdb_diagnostics-20070905041904-9d9r1qnt1j9qiwiz-1
=== modified file 'config/events.d/10.interface'
--- a/config/events.d/10.interface 2007-09-12 03:23:36 +0000
+++ b/config/events.d/10.interface 2007-09-13 00:24:48 +0000
@@ -20,6 +20,37 @@
exit 0
}
+################################################
+# kill off any TCP connections with the given IP
+kill_tcp_connections() {
+ _IP="$1"
+ _failed=0
+ _killcount=0
+ netstat -tn |egrep "^tcp.*\s+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' |
+ while read dest src; do
+ srcip=`echo $src | cut -d: -f1`
+ srcport=`echo $src | cut -d: -f2`
+ destip=`echo $dest | cut -d: -f1`
+ destport=`echo $dest | cut -d: -f2`
+ ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
+ _killcount=`expr $_killcount + 1`
+ done
+ [ _failed = 0 ] || {
+ echo "Failed to send killtcp control"
+ return;
+ }
+ _count=0
+ while netstat -tn | grep $_IP > /dev/null; do
+ sleep 1
+ _count=`expr $_count + 1`
+ [ $_count -gt 5 ] && {
+ echo "Timed out killing tcp connections for IP $_IP"
+ return;
+ }
+ done
+ echo "`date` killed $_killcount TCP connections to released IP $_IP"
+}
+
case $cmd in
#############################
# called when ctdbd starts up
@@ -29,11 +60,6 @@
[ -f /proc/sys/net/ipv4/conf/all/arp_filter ] && {
echo 1 > /proc/sys/net/ipv4/conf/all/arp_filter
}
- # make sure we don't reply to arp requests for IPs we have moved to scope
- # host on loopback
- [ -f /proc/sys/net/ipv4/conf/all/arp_ignore ] && {
- echo 3 > /proc/sys/net/ipv4/conf/all/arp_ignore
- }
;;
@@ -53,11 +79,12 @@
echo "`/bin/date` Failed to bringup interface $iface"
exit 1
}
- /sbin/ip addr del $ip/32 dev lo >/dev/null 2>/dev/null
/sbin/ip addr add $ip/$maskbits dev $iface || {
echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
exit 1
}
+ # cope with the script being killed while we have the interface blocked
+ /sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
# flush our route cache
echo 1 > /proc/sys/net/ipv4/route/flush
@@ -71,16 +98,32 @@
echo "`/bin/date` must supply interface, IP and maskbits"
exit 1
fi
+
+ # releasing an IP is a bit more complex than it seems. Once the IP
+ # is released, any open tcp connections to that IP on this host will end
+ # up being stuck. Some of them (such as NFS connections) will be unkillable
+ # so we need to use the killtcp ctdb function to kill them off. We also
+ # need to make sure that no new connections get established while we are
+ # doing this! So what we do is this:
+ # 1) firewall this IP, so no new external packets arrive for it
+ # 2) use netstat -tn to find existing connections, and kill them
+ # 3) remove the IP from the interface
+ # 4) remove the firewall rule
iface=$1
ip=$2
maskbits=$3
- /sbin/ip addr del $ip/$maskbits dev $iface || {
+
+ failed=0
+ # we do an extra delete to cope with the script being killed
+ /sbin/iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
+ /sbin/iptables -I INPUT -i $iface -d $ip -j DROP
+ kill_tcp_connections $ip
+ /sbin/ip addr del $ip/$maskbits dev $iface || failed=1
+ /sbin/iptables -D INPUT -i $iface -d $ip -j DROP
+ [ $failed = 0 ] || {
echo "`/bin/date` Failed to del $ip on dev $iface"
exit 1
}
- # we put the IP on loopback so our killtcp code can work, this allows
- # us to avoid restarting the NFS server when we release an IP
- /sbin/ip addr add $ip/32 dev lo scope host >/dev/null 2>/dev/null
# flush our route cache
echo 1 > /proc/sys/net/ipv4/route/flush
=== modified file 'config/events.d/60.nfs'
--- a/config/events.d/60.nfs 2007-09-09 21:45:57 +0000
+++ b/config/events.d/60.nfs 2007-09-13 00:24:48 +0000
@@ -55,37 +55,7 @@
maskbits=$3
echo $ip >> /etc/ctdb/state/statd/restart
-
/bin/rm -f /etc/ctdb/state/statd/ip/$ip
-
- # RST all tcp connections to the lockmanager
- [ ! -z "$LOCKD_TCPPORT" ] && {
- # RST all tcp connections used for NLM to ensure that they do
- # not survive in ESTABLISHED state across a failover/failback
- # and create an ack storm
- netstat -tn |egrep "^tcp.*\s+$ip:${LOCKD_TCPPORT}\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
- srcip=`echo $src | cut -d: -f1`
- srcport=`echo $src | cut -d: -f2`
- destip=`echo $dest | cut -d: -f1`
- destport=`echo $dest | cut -d: -f2`
- ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1
-# ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
- done
- } > /dev/null 2>&1
-
-
- # RST the local side for all tcp connections used for NFS to ensure
- # that they do not survive in ESTABLISHED state across a
- # failover/failback and create an ack storm
- netstat -tn |egrep "^tcp.*\s+$ip:2049\s+.*ESTABLISHED" | awk '{print $4" "$5}' | while read dest src; do
- srcip=`echo $src | cut -d: -f1`
- srcport=`echo $src | cut -d: -f2`
- destip=`echo $dest | cut -d: -f1`
- destport=`echo $dest | cut -d: -f2`
- ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1
-# ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1
- done
-
exit 0
;;
=== modified file 'tools/ctdb_diagnostics'
--- a/tools/ctdb_diagnostics 2007-09-10 01:27:07 +0000
+++ b/tools/ctdb_diagnostics 2007-09-13 00:24:48 +0000
@@ -123,6 +123,7 @@
show_all "/sbin/route -n"
show_all "crontab -l"
show_all "sysctl -a"
+show_all "/sbin/iptables -L -n"
[ -d /usr/lpp/mmfs ] && {
cat <<EOF
More information about the samba-cvs
mailing list