[SCM] CTDB repository - branch master updated -
f86551580349b7f662f9a07e4eb0c1189e38e429
Ronnie Sahlberg
sahlberg at samba.org
Wed Jul 2 04:28:15 GMT 2008
The branch, master has been updated
via f86551580349b7f662f9a07e4eb0c1189e38e429 (commit)
via 423934629704683d3a3042570577fb4e04b17a6d (commit)
via e4c9e511fc5e15e0638ebb9117cb4a65ca8fda4b (commit)
from af5d9435822917d36929e667063db69e6a426d3d (commit)
http://gitweb.samba.org/?p=sahlberg/ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit f86551580349b7f662f9a07e4eb0c1189e38e429
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date: Wed Jul 2 13:55:59 2008 +1000
track both when we last started and ended a recovery.
make ctdb uptime print how long the recovery took
in the recovery daemon when we check that the public ip address
allocation on the local node is correct (we have the ips we should have
and we dont have any we shouldnt have) use ctdb uptime and check the
recovery start/stop times and make sure we dont check for ip allocation
inconsistencies during a recovery where the ip address allocation is in flux.
commit 423934629704683d3a3042570577fb4e04b17a6d
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date: Wed Jul 2 12:21:53 2008 +1000
print the opcode when an async callback detects an error
commit e4c9e511fc5e15e0638ebb9117cb4a65ca8fda4b
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date: Wed Jul 2 12:01:19 2008 +1000
update a comment to reflect that this is not always a real recovery
it can also be printed when we just do an ip reallocation
-----------------------------------------------------------------------
Summary of changes:
client/ctdb_client.c | 5 +-
include/ctdb.h | 3 +-
include/ctdb_private.h | 4 +-
include/includes.h | 1 +
lib/util/util_time.c | 6 ++
server/ctdb_recover.c | 7 +-
server/ctdb_recoverd.c | 155 +++++++++++++++++++++++++++++++++++-------------
server/ctdb_uptime.c | 5 +-
server/ctdbd.c | 3 +-
tools/ctdb.c | 17 ++++--
10 files changed, 149 insertions(+), 57 deletions(-)
Changeset truncated at 500 lines:
diff --git a/client/ctdb_client.c b/client/ctdb_client.c
index 97fae50..2158d48 100644
--- a/client/ctdb_client.c
+++ b/client/ctdb_client.c
@@ -2685,7 +2685,7 @@ static void async_callback(struct ctdb_client_control_state *state)
*/
if (state->state != CTDB_CONTROL_DONE) {
if ( !data->dont_log_errors) {
- DEBUG(DEBUG_ERR,("Async operation failed with state %d\n", state->state));
+ DEBUG(DEBUG_ERR,("Async operation failed with state %d\n opcode:%u", state->state, data->opcode));
}
data->fail_count++;
if (data->fail_callback) {
@@ -2700,7 +2700,7 @@ static void async_callback(struct ctdb_client_control_state *state)
ret = ctdb_control_recv(ctdb, state, data, &outdata, &res, NULL);
if ((ret != 0) || (res != 0)) {
if ( !data->dont_log_errors) {
- DEBUG(DEBUG_ERR,("Async operation failed with ret=%d res=%d\n", ret, (int)res));
+ DEBUG(DEBUG_ERR,("Async operation failed with ret=%d res=%d opcode=%u\n", ret, (int)res, data->opcode));
}
data->fail_count++;
if (data->fail_callback) {
@@ -2769,6 +2769,7 @@ int ctdb_client_async_control(struct ctdb_context *ctdb,
async_data->callback = client_callback;
async_data->fail_callback = fail_callback;
async_data->callback_data = callback_data;
+ async_data->opcode = opcode;
num_nodes = talloc_get_size(nodes) / sizeof(uint32_t);
diff --git a/include/ctdb.h b/include/ctdb.h
index ecbe957..985b78f 100644
--- a/include/ctdb.h
+++ b/include/ctdb.h
@@ -505,7 +505,8 @@ int ctdb_ctrl_get_server_id_list(struct ctdb_context *ctdb,
struct ctdb_uptime {
struct timeval current_time;
struct timeval ctdbd_start_time;
- struct timeval last_recovery_time;
+ struct timeval last_recovery_started;
+ struct timeval last_recovery_finished;
};
int ctdb_socket_connect(struct ctdb_context *ctdb);
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index b19012f..e5a7b80 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -360,7 +360,8 @@ enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN
struct ctdb_context {
struct event_context *ev;
struct timeval ctdbd_start_time;
- struct timeval last_recovery_time;
+ struct timeval last_recovery_started;
+ struct timeval last_recovery_finished;
uint32_t recovery_mode;
TALLOC_CTX *tickle_update_context;
TALLOC_CTX *keepalive_ctx;
@@ -1313,6 +1314,7 @@ int ctdb_set_child_logging(struct ctdb_context *ctdb);
typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
struct client_async_data {
+ enum ctdb_controls opcode;
bool dont_log_errors;
uint32_t count;
uint32_t fail_count;
diff --git a/include/includes.h b/include/includes.h
index b6133ee..b1f48b8 100644
--- a/include/includes.h
+++ b/include/includes.h
@@ -46,6 +46,7 @@ struct timeval timeval_until(const struct timeval *tv1,
const struct timeval *tv2);
_PUBLIC_ struct timeval timeval_current_ofs(uint32_t secs, uint32_t usecs);
double timeval_elapsed(struct timeval *tv);
+double timeval_delta(struct timeval *tv2, struct timeval *tv);
char **file_lines_load(const char *fname, int *numlines, TALLOC_CTX *mem_ctx);
char *hex_encode_talloc(TALLOC_CTX *mem_ctx, const unsigned char *buff_in, size_t len);
_PUBLIC_ const char **str_list_add(const char **list, const char *s);
diff --git a/lib/util/util_time.c b/lib/util/util_time.c
index 13ddfdb..be14f26 100644
--- a/lib/util/util_time.c
+++ b/lib/util/util_time.c
@@ -44,6 +44,12 @@ double timeval_elapsed(struct timeval *tv)
(tv2.tv_usec - tv->tv_usec)*1.0e-6;
}
+double timeval_delta(struct timeval *tv2, struct timeval *tv)
+{
+ return (tv2->tv_sec - tv->tv_sec) +
+ (tv2->tv_usec - tv->tv_usec)*1.0e-6;
+}
+
/**
return a timeval struct with the given elements
*/
diff --git a/server/ctdb_recover.c b/server/ctdb_recover.c
index bef9451..f71f7e9 100644
--- a/server/ctdb_recover.c
+++ b/server/ctdb_recover.c
@@ -773,7 +773,7 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo
ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
talloc_free(state);
- gettimeofday(&ctdb->last_recovery_time, NULL);
+ gettimeofday(&ctdb->last_recovery_finished, NULL);
}
/*
@@ -830,7 +830,7 @@ static void ctdb_start_recovery_callback(struct ctdb_context *ctdb, int status,
}
/*
- start a recuvery
+ run the startrecovery eventscript
*/
int32_t ctdb_control_start_recovery(struct ctdb_context *ctdb,
struct ctdb_req_control *c,
@@ -839,7 +839,8 @@ int32_t ctdb_control_start_recovery(struct ctdb_context *ctdb,
int ret;
struct recovery_callback_state *state;
- DEBUG(DEBUG_NOTICE,("Recovery has started\n"));
+ DEBUG(DEBUG_NOTICE,(__location__ " startrecovery eventscript has been invoked\n"));
+ gettimeofday(&ctdb->last_recovery_started, NULL);
state = talloc(ctdb, struct recovery_callback_state);
CTDB_NO_MEMORY(ctdb, state);
diff --git a/server/ctdb_recoverd.c b/server/ctdb_recoverd.c
index 76a6a12..16068aa 100644
--- a/server/ctdb_recoverd.c
+++ b/server/ctdb_recoverd.c
@@ -2284,6 +2284,112 @@ static void ctdb_update_pnn_count(struct event_context *ev, struct timed_event *
}
}
+/* called to check that the allocation of public ip addresses is ok.
+*/
+static int verify_ip_allocation(struct ctdb_context *ctdb, uint32_t pnn)
+{
+ TALLOC_CTX *mem_ctx = talloc_new(NULL);
+ struct ctdb_all_public_ips *ips = NULL;
+ struct ctdb_uptime *uptime1 = NULL;
+ struct ctdb_uptime *uptime2 = NULL;
+ int ret, j;
+
+ ret = ctdb_ctrl_uptime(ctdb, ctdb, CONTROL_TIMEOUT(),
+ CTDB_CURRENT_NODE, &uptime1);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, ("Unable to get uptime from local node %u\n", pnn));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ /* read the ip allocation from the local node */
+ ret = ctdb_ctrl_get_public_ips(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, mem_ctx, &ips);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, ("Unable to get public ips from local node %u\n", pnn));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ ret = ctdb_ctrl_uptime(ctdb, ctdb, CONTROL_TIMEOUT(),
+ CTDB_CURRENT_NODE, &uptime2);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR, ("Unable to get uptime from local node %u\n", pnn));
+ talloc_free(mem_ctx);
+ return -1;
+ }
+
+ /* skip the check if the startrecovery time has changed */
+ if (timeval_compare(&uptime1->last_recovery_started,
+ &uptime2->last_recovery_started) != 0) {
+ DEBUG(DEBUG_NOTICE, (__location__ " last recovery time changed while we read the public ip list. skipping public ip address check\n"));
+ return 0;
+ }
+
+ /* skip the check if the endrecovery time has changed */
+ if (timeval_compare(&uptime1->last_recovery_finished,
+ &uptime2->last_recovery_finished) != 0) {
+ DEBUG(DEBUG_NOTICE, (__location__ " last recovery time changed while we read the public ip list. skipping public ip address check\n"));
+ return 0;
+ }
+
+ /* skip the check if we have started but not finished recovery */
+ if (timeval_compare(&uptime1->last_recovery_finished,
+ &uptime1->last_recovery_started) != 1) {
+ DEBUG(DEBUG_NOTICE, (__location__ " in the middle of recovery. skipping public ip address check\n"));
+
+ return 0;
+ }
+
+ /* verify that we have the ip addresses we should have
+ and we dont have ones we shouldnt have.
+ if we find an inconsistency we set recmode to
+ active on the local node and wait for the recmaster
+ to do a full blown recovery
+ */
+ for (j=0; j<ips->num; j++) {
+ if (ips->ips[j].pnn == pnn) {
+ if (!ctdb_sys_have_ip(ips->ips[j].sin)) {
+ DEBUG(DEBUG_CRIT,("Public address '%s' is missing and we should serve this ip\n", inet_ntoa(ips->ips[j].sin.sin_addr)));
+ ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
+
+ talloc_free(mem_ctx);
+ return -1;
+ }
+ ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
+
+ talloc_free(mem_ctx);
+ return -1;
+ }
+ }
+ } else {
+ if (ctdb_sys_have_ip(ips->ips[j].sin)) {
+ DEBUG(DEBUG_CRIT,("We are still serving a public address '%s' that we should not be serving.\n", inet_ntoa(ips->ips[j].sin.sin_addr)));
+ ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
+
+ talloc_free(mem_ctx);
+ return -1;
+ }
+ ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
+ if (ret != 0) {
+ DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
+
+ talloc_free(mem_ctx);
+ return -1;
+ }
+ }
+ }
+ }
+
+ talloc_free(mem_ctx);
+ return 0;
+}
+
/*
the main monitoring loop
*/
@@ -2298,7 +2404,6 @@ static void monitor_cluster(struct ctdb_context *ctdb)
int32_t debug_level;
int i, j, ret;
struct ctdb_recoverd *rec;
- struct ctdb_all_public_ips *ips;
char c;
DEBUG(DEBUG_NOTICE,("monitor_cluster starting\n"));
@@ -2523,49 +2628,15 @@ again:
update_our_flags_on_all_nodes(ctdb, pnn, nodemap);
}
- /* verify that the public ip address allocation is consistent */
- ret = ctdb_ctrl_get_public_ips(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, mem_ctx, &ips);
- if (ret != 0) {
- DEBUG(DEBUG_ERR, ("Unable to get public ips from node %u\n", i));
+
+ /* verify that we have all ip addresses we should have and we dont
+ * have addresses we shouldnt have.
+ */
+ if (verify_ip_allocation(ctdb, pnn) != 0) {
+ DEBUG(DEBUG_ERR, (__location__ " Public IPs were inconsistent.\n"));
goto again;
}
- for (j=0; j<ips->num; j++) {
- /* verify that we have the ip addresses we should have
- and we dont have ones we shouldnt have.
- if we find an inconsistency we set recmode to
- active on the local node and wait for the recmaster
- to do a full blown recovery
- */
- if (ips->ips[j].pnn == pnn) {
- if (!ctdb_sys_have_ip(ips->ips[j].sin)) {
- DEBUG(DEBUG_CRIT,("Public address '%s' is missing and we should serve this ip\n", inet_ntoa(ips->ips[j].sin.sin_addr)));
- ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
- goto again;
- }
- ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
- goto again;
- }
- }
- } else {
- if (ctdb_sys_have_ip(ips->ips[j].sin)) {
- DEBUG(DEBUG_CRIT,("We are still serving a public address '%s' that we should not be serving.\n", inet_ntoa(ips->ips[j].sin.sin_addr)));
- ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to freeze node due to public ip address mismatches\n"));
- goto again;
- }
- ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, CTDB_RECOVERY_ACTIVE);
- if (ret != 0) {
- DEBUG(DEBUG_ERR,(__location__ " Failed to activate recovery mode due to public ip address mismatches\n"));
- goto again;
- }
- }
- }
- }
+
/* if we are not the recmaster then we do not need to check
if recovery is needed
diff --git a/server/ctdb_uptime.c b/server/ctdb_uptime.c
index 083f6b5..6bc8158 100644
--- a/server/ctdb_uptime.c
+++ b/server/ctdb_uptime.c
@@ -35,8 +35,9 @@ int32_t ctdb_control_uptime(struct ctdb_context *ctdb, TDB_DATA *outdata)
CTDB_NO_MEMORY(ctdb, uptime);
gettimeofday(&uptime->current_time, NULL);
- uptime->ctdbd_start_time = ctdb->ctdbd_start_time;
- uptime->last_recovery_time = ctdb->last_recovery_time;
+ uptime->ctdbd_start_time = ctdb->ctdbd_start_time;
+ uptime->last_recovery_started = ctdb->last_recovery_started;
+ uptime->last_recovery_finished = ctdb->last_recovery_finished;
outdata->dsize = sizeof(struct ctdb_uptime);
outdata->dptr = (uint8_t *)uptime;
diff --git a/server/ctdbd.c b/server/ctdbd.c
index c21d434..aa5253a 100644
--- a/server/ctdbd.c
+++ b/server/ctdbd.c
@@ -174,7 +174,8 @@ int main(int argc, const char *argv[])
DEBUG(DEBUG_NOTICE,("Starting CTDB daemon\n"));
gettimeofday(&ctdb->ctdbd_start_time, NULL);
- gettimeofday(&ctdb->last_recovery_time, NULL);
+ gettimeofday(&ctdb->last_recovery_started, NULL);
+ gettimeofday(&ctdb->last_recovery_finished, NULL);
ctdb->recovery_mode = CTDB_RECOVERY_NORMAL;
ctdb->recovery_master = (uint32_t)-1;
ctdb->upcalls = &ctdb_upcalls;
diff --git a/tools/ctdb.c b/tools/ctdb.c
index e9a109f..1d77707 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -248,11 +248,14 @@ static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv
}
if (options.machinereadable){
- printf(":Current Node Time:Ctdb Start Time:Last Recovery Time:\n");
- printf(":%u:%u:%u:\n",
+ printf(":Current Node Time:Ctdb Start Time:Last Recovery Time:Last Recovery Duration:\n");
+ printf(":%u:%u:%u:%lf\n",
(unsigned int)uptime->current_time.tv_sec,
(unsigned int)uptime->ctdbd_start_time.tv_sec,
- (unsigned int)uptime->last_recovery_time.tv_sec);
+ (unsigned int)uptime->last_recovery_finished.tv_sec,
+ timeval_delta(&uptime->last_recovery_finished,
+ &uptime->last_recovery_started)
+ );
return 0;
}
@@ -268,7 +271,7 @@ static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv
days = tmp;
printf("Ctdbd start time : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->ctdbd_start_time.tv_sec));
- tmp = uptime->current_time.tv_sec - uptime->last_recovery_time.tv_sec;
+ tmp = uptime->current_time.tv_sec - uptime->last_recovery_finished.tv_sec;
seconds = tmp%60;
tmp /= 60;
minutes = tmp%60;
@@ -276,7 +279,11 @@ static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv
hours = tmp%24;
tmp /= 24;
days = tmp;
- printf("Time of last recovery : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_time.tv_sec));
+ printf("Time of last recovery : (%03d %02d:%02d:%02d) %s", days, hours, minutes, seconds, ctime(&uptime->last_recovery_finished.tv_sec));
+
+ printf("Duration of last recovery : %lf seconds\n",
+ timeval_delta(&uptime->last_recovery_finished,
+ &uptime->last_recovery_started));
return 0;
}
--
CTDB repository
More information about the samba-cvs
mailing list