[SCM] CTDB repository - branch master updated - ctdb-1.0.114-181-gd148e7a
Ronnie Sahlberg
sahlberg at samba.org
Fri Jul 30 00:57:01 MDT 2010
The branch, master has been updated
via d148e7a7cb840febbdf56ba2e39c314cc2d7ac24 (commit)
via 6549e9b01538998d51a5f72bfc569776d232b024 (commit)
via b2362cc7773bb08c7dfdaf2c87d4b59460686659 (commit)
via 20ea31e4ed893eb58cb2efa0b6fb13bcf4031918 (commit)
from 8024d9e2d589bfe4dee1cb9a79bec663738cb7fa (commit)
http://gitweb.samba.org/?p=sahlberg/ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit d148e7a7cb840febbdf56ba2e39c314cc2d7ac24
Author: Martin Schwenke <martin at meltin.net>
Date: Fri Jul 30 16:45:36 2010 +1000
Testing: Add Python IP allocation simulation.
Includes simulation module and example scenarios. This allows you to
test and perhaps tweak an algorithm that should be the same as the
current CTDB IP reallocation one.
Signed-off-by: Martin Schwenke <martin at meltin.net>
commit 6549e9b01538998d51a5f72bfc569776d232b024
Author: Martin Schwenke <martin at meltin.net>
Date: Mon Jul 26 16:22:59 2010 +1000
Optimise 61.nfstickle to write the tickles more efficiently.
Currently the file for each IP address is reopened to append the
details of each source socket.
This optimisation puts all the logic into awk, including the matching
of output lines from netstat. The source sockets for each for each
destination IP are written into an array entry and then each array
entry is written to the corresponding file in a single operation.
Signed-off-by: Martin Schwenke <martin at meltin.net>
commit b2362cc7773bb08c7dfdaf2c87d4b59460686659
Author: Martin Schwenke <martin at meltin.net>
Date: Mon Jun 7 12:03:25 2010 +1000
Test suite: handle extra lines in statistics output.
Signed-off-by: Martin Schwenke <martin at meltin.net>
commit 20ea31e4ed893eb58cb2efa0b6fb13bcf4031918
Author: Martin Schwenke <martin at meltin.net>
Date: Mon Jun 7 12:29:31 2010 +1000
Test suite: handle change to disconnected node error message.
Signed-off-by: Martin Schwenke <martin at meltin.net>
-----------------------------------------------------------------------
Summary of changes:
.gitignore | 1 +
config/events.d/61.nfstickle | 17 +-
tests/simple/14_ctdb_statistics.sh | 2 +-
..._ctdb_config_check_error_on_unreachable_ctdb.sh | 2 +-
tests/takeover/README | 3 +
tests/takeover/ctdb_takeover.py | 442 ++++++++++++++++++++
tests/takeover/mgmt_simple.py | 20 +
tests/takeover/node_pool_extra.py | 29 ++
tests/takeover/node_pool_simple.py | 24 +
tests/takeover/nondet_path_01.py | 24 +
10 files changed, 556 insertions(+), 8 deletions(-)
create mode 100644 tests/takeover/README
create mode 100755 tests/takeover/ctdb_takeover.py
create mode 100755 tests/takeover/mgmt_simple.py
create mode 100755 tests/takeover/node_pool_extra.py
create mode 100755 tests/takeover/node_pool_simple.py
create mode 100755 tests/takeover/nondet_path_01.py
Changeset truncated at 500 lines:
diff --git a/.gitignore b/.gitignore
index 69d8093..a4cc47e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,4 @@ test.db
tests/bin
tests/events.d/00.ctdb_test_trigger
tests/var
+tests/takeover/ctdb_takeover.pyc
diff --git a/config/events.d/61.nfstickle b/config/events.d/61.nfstickle
index deb7966..3bfef4f 100755
--- a/config/events.d/61.nfstickle
+++ b/config/events.d/61.nfstickle
@@ -41,12 +41,17 @@ case "$1" in
mydir=$NFS_TICKLE_SHARED_DIRECTORY/`hostname`
rm -f $mydir/*
# record our connections to shared storage
- netstat -tn |egrep '^tcp[[:space:]]+[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9\.]+:2049.*ESTABLISHED' |
- awk '{print $4" "$5}' |
- while read dest src; do
- ip=${dest%:*}
- echo $src >> $mydir/$ip
- done
+ netstat -tn |
+ awk -v mydir="$mydir" '
+$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ /:2049$/ {
+ destip = gensub(/:2049$/, "", 1, $4);
+ c[destip] = c[destip] (c[destip] ? "\n" : "" ) $5;
+}
+END {
+ for (ip in c) {
+ print c[ip] > mydir "/" ip
+ }
+}'
;;
*)
diff --git a/tests/simple/14_ctdb_statistics.sh b/tests/simple/14_ctdb_statistics.sh
index 58c76be..9a95a83 100755
--- a/tests/simple/14_ctdb_statistics.sh
+++ b/tests/simple/14_ctdb_statistics.sh
@@ -33,7 +33,7 @@ set -e
cluster_is_healthy
-pattern='^(CTDB version 1|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+([[:alpha:]_]+_latency|max_reclock_[[:alpha:]]+)[[:space:]]+[[:digit:]-]+\.[[:digit:]]+[[:space:]]sec)$'
+pattern='^(CTDB version 1|Current time of statistics[[:space:]]*:.*|Statistics collected since[[:space:]]*:.*|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+([[:alpha:]_]+_latency|max_reclock_[[:alpha:]]+)[[:space:]]+[[:digit:]-]+\.[[:digit:]]+[[:space:]]sec)$'
try_command_on_node -v 1 "$CTDB statistics"
diff --git a/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index 5c5298a..da5a49b 100755
--- a/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -54,7 +54,7 @@ try_command_on_node $test_node $CTDB shutdown
wait_until_node_has_status $test_node disconnected
-pat="ctdb_control error: 'ctdb_control to disconnected node'|Node $test_node is DISCONNECTED"
+pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error: 'node is disconnected'|Node $test_node is DISCONNECTED"
for i in ip disable enable "ban 0" unban listvars ; do
try_command_on_node -v 0 ! $CTDB $i -n $test_node
diff --git a/tests/takeover/README b/tests/takeover/README
new file mode 100644
index 0000000..73815c7
--- /dev/null
+++ b/tests/takeover/README
@@ -0,0 +1,3 @@
+This contains a Python simulation of CTDB's IP reallocation algorithm.
+
+It is useful for experimenting with improvements.
diff --git a/tests/takeover/ctdb_takeover.py b/tests/takeover/ctdb_takeover.py
new file mode 100755
index 0000000..19608a3
--- /dev/null
+++ b/tests/takeover/ctdb_takeover.py
@@ -0,0 +1,442 @@
+#!/usr/bin/env python
+
+# ctdb ip takeover code
+
+# Copyright (C) Ronnie Sahlberg 2007
+# Copyright (C) Andrew Tridgell 2007
+#
+# Python version (C) Martin Schwenke 2010
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import sys
+from optparse import OptionParser
+import copy
+import random
+
+def process_args():
+ usage = "usage: %prog [options]"
+
+ parser = OptionParser(usage=usage)
+
+ parser.add_option("--nd",
+ action="store_false", dest="deterministic_public_ips",
+ default=True,
+ help="turn off deterministic_public_ips")
+ parser.add_option("--ni",
+ action="store_true", dest="no_ip_failback", default=False,
+ help="turn on no_ip_failback")
+ parser.add_option("-v", "--verbose",
+ action="store_true", dest="verbose", default=False,
+ help="print information and actions taken to stdout")
+ parser.add_option("-d", "--diff",
+ action="store_true", dest="diff", default=False,
+ help="after each recovery show IP address movements")
+ parser.add_option("-n", "--no-print",
+ action="store_false", dest="show", default=True,
+ help="after each recovery don't print IP address layout")
+ parser.add_option("--hack",
+ action="store", type="int", dest="hack", default=0,
+ help="apply a hack (see the code!!!)")
+ parser.add_option("-r", "--retries",
+ action="store", type="int", dest="retries", default=5,
+ help="number of retry loops for rebalancing")
+ parser.add_option("-i", "--iterations",
+ action="store", type="int", dest="iterations",
+ default=1000,
+ help="number of iterations to run in test")
+ parser.add_option("-x", "--exit",
+ action="store_true", dest="exit", default=False,
+ help="exit on the 1st gratuitous IP move")
+
+ (options, args) = parser.parse_args()
+
+ if len(args) != 0:
+ parser.error("too many argumentss")
+
+ return options
+
+def print_begin(t):
+ print "=" * 40
+ print "%s:" % (t)
+
+def print_end():
+ print "-" * 40
+
+def verbose_begin(t):
+ if options.verbose:
+ print_begin(t)
+
+def verbose_end():
+ if options.verbose:
+ print_end()
+
+def verbose_print(t):
+ if options.verbose:
+ if not type(t) == list:
+ t = [t]
+ if t != []:
+ print "\n".join([str(i) for i in t])
+
+
+class Node(object):
+ def __init__(self, public_addresses):
+ self.public_addresses = set(public_addresses)
+ self.current_addresses = set()
+ self.healthy = True
+
+ def can_node_serve_ip(self, ip):
+ return ip in self.public_addresses
+
+ def node_ip_coverage(self):
+ return len(self.current_addresses)
+
+class Cluster(object):
+ def __init__(self):
+ global options
+
+ self.nodes = []
+ self.deterministic_public_ips = options.deterministic_public_ips
+ self.no_ip_failback = options.no_ip_failback
+ self.all_public_ips = set()
+
+ self.ip_moves = []
+ self.grat_ip_moves = []
+ self.imbalance = []
+
+ def __str__(self):
+ return "\n".join(["%2d %s %s" %
+ (i,
+ "*" if len(n.public_addresses) == 0 else \
+ (" " if n.healthy else "#"),
+ sorted(list(n.current_addresses)))
+ for (i, n) in enumerate(self.nodes)])
+
+ def print_statistics(self):
+ print_begin("STATISTICS")
+ print "Total IP moves: %6d" % sum(self.ip_moves)
+ print "Gratuitous IP moves: %6d" % sum(self.grat_ip_moves)
+ print "Max imbalance: %6d" % max(self.imbalance)
+ print_end()
+
+ def find_pnn_with_ip(self, ip):
+ for (i, n) in enumerate(self.nodes):
+ if ip in n.current_addresses:
+ return i
+ return -1
+
+ def quietly_remove_ip(self, ip):
+ # Remove address from old node.
+ old = self.find_pnn_with_ip(ip)
+ if old != -1:
+ self.nodes[old].current_addresses.remove(ip)
+
+ def add_node(self, node):
+ self.nodes.append(node)
+ self.all_public_ips |= node.public_addresses
+
+ def healthy(self, *pnns):
+ global options
+
+ verbose_begin("HEALTHY")
+
+ for pnn in pnns:
+ self.nodes[pnn].healthy = True
+ verbose_print(pnn)
+
+ verbose_end()
+
+ def unhealthy(self, *pnns):
+
+ verbose_begin("UNHEALTHY")
+
+ for pnn in pnns:
+ self.nodes[pnn].healthy = False
+ verbose_print(pnn)
+
+ verbose_end()
+
+ def do_something_random(self):
+
+
+ """Make a random node healthy or unhealthy.
+
+ If all nodes are healthy or unhealthy, then invert one of
+ them. Otherwise, there's a 1/4 chance of making another node
+ unhealthy."""
+
+ num_nodes = len(self.nodes)
+ healthy_nodes = [n for n in self.nodes if n.healthy]
+ num_healthy = len(healthy_nodes)
+
+ if num_nodes == num_healthy:
+ self.unhealthy(random.randint(0, num_nodes-1))
+ elif num_healthy == 0:
+ self.healthy(random.randint(0, num_nodes-1))
+ elif random.randint(1, 4) == 1:
+ self.unhealthy(self.nodes.index(random.choice(healthy_nodes)))
+ else:
+ self.healthy(self.nodes.index(random.choice(list(set(self.nodes) - set(healthy_nodes)))))
+
+ def random_iterations(self):
+ i = 1
+ while i <= options.iterations:
+ print_begin("EVENT %d" % i)
+ print_end()
+ self.do_something_random()
+ if self.recover() and options.exit > 0:
+ break
+ i += 1
+
+ self.print_statistics()
+
+
+ def diff(self, prev):
+ """Calculate differences in IP assignments between self and prev.
+
+ Gratuitous IP moves (from a healthy node to a healthy node)
+ are prefix by !!. Any gratuitous IP moves cause this function
+ to return False. If there are no gratuitous moves then it
+ will return True."""
+
+ ip_moves = 0
+ grat_ip_moves = 0
+ imbalance = 0
+ details = []
+
+ for (new, n) in enumerate(self.nodes):
+ for ip in n.current_addresses:
+ old = prev.find_pnn_with_ip(ip)
+ if old != new:
+ ip_moves += 1
+ if old != -1 and \
+ prev.nodes[new].healthy and \
+ self.nodes[new].healthy and \
+ self.nodes[old].healthy and \
+ prev.nodes[old].healthy:
+ prefix = "!!"
+ grat_ip_moves += 1
+ else:
+ prefix = " "
+ details.append("%s %s: %d -> %d" %
+ (prefix, ip, old, new))
+
+ return (ip_moves, grat_ip_moves, imbalance, details)
+
+ def find_least_loaded_node(self, ip):
+ """Just like find_takeover_node but doesn't care about health."""
+ pnn = -1
+ min = 0
+ for (i, n) in enumerate(self.nodes):
+ if not n.can_node_serve_ip(ip):
+ continue
+
+ num = n.node_ip_coverage()
+
+ if (pnn == -1):
+ pnn = i
+ min = num
+ else:
+ if num < min:
+ pnn = i
+ min = num
+
+ if pnn == -1:
+ print "Could not find node to take over public address", ip
+ return False
+
+ self.nodes[pnn].current_addresses.add(ip)
+
+ verbose_print("%s -> %d" % (ip, pnn))
+ return True
+
+ def find_takeover_node(self, ip):
+
+ pnn = -1
+ min = 0
+ for (i, n) in enumerate(self.nodes):
+ if not n.healthy:
+ continue
+
+ if not n.can_node_serve_ip(ip):
+ continue
+
+ num = n.node_ip_coverage()
+
+ if (pnn == -1):
+ pnn = i
+ min = num
+ else:
+ if num < min:
+ pnn = i
+ min = num
+
+ if pnn == -1:
+ print "Could not find node to take over public address", ip
+ return False
+
+ self.nodes[pnn].current_addresses.add(ip)
+
+ verbose_print("%s -> %d" % (ip, pnn))
+ return True
+
+ def ctdb_takeover_run(self):
+
+ global options
+
+ # Don't bother with the num_healthy stuff. It is an
+ # irrelevant detail.
+
+ # We just keep the allocate IPs in the current_addresses field
+ # of the node. This needs to readable, not efficient!
+
+ if self.deterministic_public_ips:
+ # Remap everything.
+ addr_list = sorted(list(self.all_public_ips))
+ for (i, ip) in enumerate(addr_list):
+ if options.hack == 1:
+ self.quietly_remove_ip(ip)
+ self.find_least_loaded_node(ip)
+ elif options.hack == 2:
+ pnn = i % len(self.nodes)
+ if ip in self.nodes[pnn].public_addresses:
+ self.quietly_remove_ip(ip)
+ # Add addresses to new node.
+ self.nodes[pnn].current_addresses.add(ip)
+ verbose_print("%s -> %d" % (ip, pnn))
+ else:
+ self.quietly_remove_ip(ip)
+ # Add addresses to new node.
+ pnn = i % len(self.nodes)
+ self.nodes[pnn].current_addresses.add(ip)
+ verbose_print("%s -> %d" % (ip, pnn))
+
+ # Remove public addresses from unhealthy nodes.
+ for (pnn, n) in enumerate(self.nodes):
+ if not n.healthy:
+ verbose_print(["%s <- %d" % (ip, pnn)
+ for ip in n.current_addresses])
+ n.current_addresses = set()
+
+ # If a node can't serve an assigned address then remove it.
+ for n in self.nodes:
+ verbose_print(["%s <- %d" % (ip, pnn)
+ for ip in n.current_addresses - n.public_addresses])
+ n.current_addresses &= n.public_addresses
+
+ # We'll only retry the balancing act up to 5 times.
+ retries = 0
+ should_loop = True
+ while should_loop:
+ should_loop = False
+
+ assigned = set([ip for n in self.nodes for ip in n.current_addresses])
+ unassigned = sorted(list(self.all_public_ips - assigned))
+
+ for ip in unassigned:
+ self.find_takeover_node(ip)
+
+ if self.no_ip_failback:
+ break
+
+ assigned = sorted([ip
+ for n in self.nodes
+ for ip in n.current_addresses])
+ for ip in assigned:
+
+ maxnode = -1
+ minnode = -1
+ for (i, n) in enumerate(self.nodes):
+ if not n.healthy:
+ continue
+
+ if not n.can_node_serve_ip(ip):
+ continue
+
+ num = n.node_ip_coverage()
+
+ if maxnode == -1:
+ maxnode = i
+ maxnum = num
+ else:
+ if num > maxnum:
+ maxnode = i
+ maxnum = num
+ if minnode == -1:
+ minnode = i
+ minnum = num
+ else:
+ if num < minnum:
+ minnode = i
+ minnum = num
+
+ if maxnode == -1:
+ print "Could not maxnode. May not be able to serve ip", ip
+ continue
+
+ if self.deterministic_public_ips:
+ continue
+
+ if maxnum > minnum + 1 and retries < options.retries:
+ # Remove the 1st ip from maxnode
+ t = sorted(list(self.nodes[maxnode].current_addresses))
+ realloc = t[0]
+ verbose_print("%s <- %d" % (realloc, maxnode))
+ self.nodes[maxnode].current_addresses.remove(realloc)
+ retries += 1
+ # Redo the outer loop.
+ should_loop = True
+ break
+
+ def recover(self):
+ global options, prev
+
+ verbose_begin("TAKEOVER")
+
+ self.ctdb_takeover_run()
+
+ verbose_end()
+
+ grat_ip_moves = 0
+
+ if prev is not None:
+ (ip_moves, grat_ip_moves, imbalance, details) = self.diff(prev)
+ self.ip_moves.append(ip_moves)
+ self.grat_ip_moves.append(grat_ip_moves)
+ self.imbalance.append(imbalance)
+
+ if options.diff:
+ print_begin("DIFF")
--
CTDB repository
More information about the samba-cvs
mailing list