[SCM] CTDB repository - branch master updated - ctdb-1.0.104-8-g29d2ee8
Ronnie Sahlberg
sahlberg at samba.org
Tue Nov 17 18:43:02 MST 2009
The branch, master has been updated
via 29d2ee8d9c6c6f36b2334480f646d6db209f370e (commit)
from 354c0edacf2d6cec5b295e139d4fec618bad1b06 (commit)
http://gitweb.samba.org/?p=sahlberg/ctdb.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 29d2ee8d9c6c6f36b2334480f646d6db209f370e
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date: Wed Nov 18 12:44:18 2009 +1100
add an in memory ringbuffer where we store the last 500000 log entries regardless of log level.
add commandt to extract this in memory buffer and to clear it
-----------------------------------------------------------------------
Summary of changes:
Makefile.in | 3 +-
common/ctdb_logging.c | 164 ++++++++++++++++++++++++++++++++++++++++++++++++
include/ctdb_private.h | 20 ++++++
include/includes.h | 2 +-
lib/util/debug.c | 2 +-
lib/util/debug.h | 1 +
server/ctdb_control.c | 7 ++
server/ctdb_logging.c | 5 ++
tools/ctdb.c | 111 ++++++++++++++++++++++++++++++++
9 files changed, 312 insertions(+), 3 deletions(-)
create mode 100644 common/ctdb_logging.c
Changeset truncated at 500 lines:
diff --git a/Makefile.in b/Makefile.in
index 4b2d100..df80e7f 100755
--- a/Makefile.in
+++ b/Makefile.in
@@ -39,7 +39,8 @@ UTIL_OBJ = lib/util/idtree.o lib/util/db_wrap.o lib/util/strlist.o lib/util/util
CTDB_COMMON_OBJ = common/ctdb_io.o common/ctdb_util.o \
common/ctdb_ltdb.o common/ctdb_message.o common/cmdline.o \
- lib/util/debug.o common/rb_tree.o @CTDB_SYSTEM_OBJ@ common/system_common.o
+ lib/util/debug.o common/rb_tree.o @CTDB_SYSTEM_OBJ@ common/system_common.o \
+ common/ctdb_logging.c
CTDB_TCP_OBJ = tcp/tcp_connect.o tcp/tcp_io.o tcp/tcp_init.o
diff --git a/common/ctdb_logging.c b/common/ctdb_logging.c
new file mode 100644
index 0000000..3183fe8
--- /dev/null
+++ b/common/ctdb_logging.c
@@ -0,0 +1,164 @@
+/*
+ ctdb logging code
+
+ Copyright (C) Ronnie Sahlberg 2009
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "../include/ctdb_private.h"
+#include "../include/ctdb.h"
+
+struct ctdb_log_entry {
+ int32_t level;
+ struct timeval t;
+ char *message;
+};
+
+#define MAX_LOG_ENTRIES 500000
+static int first_entry;
+static int last_entry;
+
+static struct ctdb_log_entry log_entries[MAX_LOG_ENTRIES];
+
+/*
+ * this function logs all messages for all levels to a ringbuffer
+ */
+static void log_ringbuffer_v(const char *format, va_list ap)
+{
+ int ret;
+ char *s = NULL;
+
+ if (log_entries[last_entry].message != NULL) {
+ free(log_entries[last_entry].message);
+ log_entries[last_entry].message = NULL;
+ }
+
+ ret = vasprintf(&s, format, ap);
+ if (ret == -1) {
+ return;
+ }
+
+ log_entries[last_entry].level = this_log_level;
+ log_entries[last_entry].t = timeval_current();
+ log_entries[last_entry].message = s;
+
+ last_entry++;
+ if (last_entry >= MAX_LOG_ENTRIES) {
+ last_entry = 0;
+ }
+ if (first_entry == last_entry) {
+ first_entry++;
+ }
+ if (first_entry >= MAX_LOG_ENTRIES) {
+ first_entry = 0;
+ }
+}
+
+void log_ringbuffer(const char *format, ...)
+{
+ va_list ap;
+
+ va_start(ap, format);
+ log_ringbuffer_v(format, ap);
+ va_end(ap);
+}
+
+
+
+static void ctdb_collect_log(struct ctdb_context *ctdb, struct ctdb_get_log_addr *log_addr)
+{
+ char *buf = talloc_size(NULL, 0);
+ struct ctdb_log_entry_wire *log_entry;
+ uint32_t old_size, len;
+ TDB_DATA data;
+
+ DEBUG(DEBUG_INFO,("Marshalling log entries\n"));
+ while (first_entry != last_entry) {
+ int slen = strlen(log_entries[first_entry].message);
+
+ if (log_entries[first_entry].level > log_addr->level) {
+ first_entry++;
+ if (first_entry >= MAX_LOG_ENTRIES) {
+ first_entry = 0;
+ }
+ continue;
+ }
+
+ len = offsetof(struct ctdb_log_entry_wire, message) + slen + 1;
+ /* pad it to uint42 */
+ len = (len+3)&0xfffffffc;
+
+ old_size = talloc_get_size(buf);
+ buf = talloc_realloc_size(NULL, buf, old_size + len);
+
+ log_entry = (struct ctdb_log_entry_wire *)&buf[old_size];
+ log_entry->level = log_entries[first_entry].level;
+ log_entry->t = log_entries[first_entry].t;
+ log_entry->message_len = slen;
+ memcpy(log_entry->message, log_entries[first_entry].message, slen);
+ log_entry->message[slen] = 0;
+
+ first_entry++;
+ if (first_entry >= MAX_LOG_ENTRIES) {
+ first_entry = 0;
+ }
+ }
+
+ data.dptr = (uint8_t *)buf;
+ data.dsize = talloc_get_size(buf);
+ DEBUG(DEBUG_INFO,("Marshalling log entries into a blob of %d bytes\n", (int)data.dsize));
+
+ DEBUG(DEBUG_INFO,("Send log to %d:%d\n", (int)log_addr->pnn, (int)log_addr->srvid));
+ ctdb_send_message(ctdb, log_addr->pnn, log_addr->srvid, data);
+}
+
+int32_t ctdb_control_get_log(struct ctdb_context *ctdb, TDB_DATA addr)
+{
+ struct ctdb_get_log_addr *log_addr = (struct ctdb_get_log_addr *)addr.dptr;
+ pid_t child;
+
+ /* spawn a child process to marshall the huge log blob and send it back
+ to the ctdb tool using a MESSAGE
+ */
+ child = fork();
+ if (child == (pid_t)-1) {
+ DEBUG(DEBUG_ERR,("Failed to fork a log collector child\n"));
+ return -1;
+ }
+
+ if (child == 0) {
+ if (switch_from_server_to_client(ctdb) != 0) {
+ DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch log collector child into client mode.\n"));
+ _exit(1);
+ }
+ ctdb_collect_log(ctdb, log_addr);
+ _exit(0);
+ }
+
+ return 0;
+}
+
+
+int32_t ctdb_control_clear_log(struct ctdb_context *ctdb)
+{
+ first_entry = 0;
+ last_entry = 0;
+
+ return 0;
+}
+
diff --git a/include/ctdb_private.h b/include/ctdb_private.h
index 3fcfdfa..48fb29b 100644
--- a/include/ctdb_private.h
+++ b/include/ctdb_private.h
@@ -622,6 +622,8 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS = 0,
CTDB_CONTROL_REGISTER_NOTIFY = 114,
CTDB_CONTROL_DEREGISTER_NOTIFY = 115,
CTDB_CONTROL_TRANS2_ACTIVE = 116,
+ CTDB_CONTROL_GET_LOG = 117,
+ CTDB_CONTROL_CLEAR_LOG = 118,
};
/*
@@ -1520,4 +1522,22 @@ int32_t ctdb_control_deregister_notify(struct ctdb_context *ctdb, uint32_t clien
int start_syslog_daemon(struct ctdb_context *ctdb);
+/* Where to send the log messages back to */
+struct ctdb_get_log_addr {
+ uint32_t pnn;
+ uint64_t srvid;
+ int32_t level;
+};
+
+/* wire data for log entries, padded to uint32 */
+struct ctdb_log_entry_wire {
+ int32_t level;
+ struct timeval t;
+ int32_t message_len;
+ char message[1];
+};
+
+int32_t ctdb_control_get_log(struct ctdb_context *ctdb, TDB_DATA addr);
+int32_t ctdb_control_clear_log(struct ctdb_context *ctdb);
+
#endif
diff --git a/include/includes.h b/include/includes.h
index 744d117..4b4d242 100644
--- a/include/includes.h
+++ b/include/includes.h
@@ -28,7 +28,7 @@ enum debug_level {
DEBUG_DEBUG = 4,
};
-#define DEBUG(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug x; }} while (0)
+#define DEBUG(lvl, x) do { this_log_level = (lvl); log_ringbuffer x; if ((lvl) <= LogLevel) { do_debug x; }} while (0)
#define _PUBLIC_
diff --git a/lib/util/debug.c b/lib/util/debug.c
index 63ca03b..6597570 100644
--- a/lib/util/debug.c
+++ b/lib/util/debug.c
@@ -90,7 +90,7 @@ void do_debug_add(const char *format, ...)
}
#define DEBUGLVL(lvl) ((lvl) <= LogLevel)
-#define DEBUG(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug x; }} while (0)
+#define DEBUG(lvl, x) do { this_log_level = (lvl); log_ringbuffer x; if ((lvl) <= LogLevel) { do_debug x; }} while (0)
#define DEBUGADD(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug_add x; }} while (0)
static void print_asc(int level, const uint8_t *buf, size_t len)
diff --git a/lib/util/debug.h b/lib/util/debug.h
index a7d8978..d91f43b 100644
--- a/lib/util/debug.h
+++ b/lib/util/debug.h
@@ -19,5 +19,6 @@
void (*do_debug_v)(const char *, va_list ap);
void (*do_debug_add_v)(const char *, va_list ap);
+void log_ringbuffer(const char *format, ...);
void do_debug(const char *format, ...) PRINTF_ATTRIBUTE(1, 2);
void dump_data(int level, const uint8_t *buf1, size_t len);
diff --git a/server/ctdb_control.c b/server/ctdb_control.c
index bf82e51..168e1b5 100644
--- a/server/ctdb_control.c
+++ b/server/ctdb_control.c
@@ -564,6 +564,13 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_client_notify_deregister));
return ctdb_control_deregister_notify(ctdb, client_id, indata);
+ case CTDB_CONTROL_GET_LOG:
+ CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_get_log_addr));
+ return ctdb_control_get_log(ctdb, indata);
+
+ case CTDB_CONTROL_CLEAR_LOG:
+ return ctdb_control_clear_log(ctdb);
+
default:
DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode));
return -1;
diff --git a/server/ctdb_logging.c b/server/ctdb_logging.c
index 16f728b..5bb39e3 100644
--- a/server/ctdb_logging.c
+++ b/server/ctdb_logging.c
@@ -448,3 +448,8 @@ int ctdb_set_child_logging(struct ctdb_context *ctdb)
return 0;
}
+
+
+
+
+
diff --git a/tools/ctdb.c b/tools/ctdb.c
index 838efdb..e89640a 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -2354,6 +2354,115 @@ static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
}
+static void log_handler(struct ctdb_context *ctdb, uint64_t srvid,
+ TDB_DATA data, void *private_data)
+{
+ struct ctdb_log_entry_wire *entry;
+ int size;
+ struct tm *tm;
+ char tbuf[100];
+
+ DEBUG(DEBUG_ERR,("Log data received\n"));
+ while (data.dsize > 0) {
+ entry = (struct ctdb_log_entry_wire *)data.dptr;
+ size = offsetof(struct ctdb_log_entry_wire, message) + entry->message_len + 1;
+ size = (size+3)&0xfffffffc;
+
+ tm = localtime(&entry->t.tv_sec);
+
+ strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
+
+ printf("%s:%s %s", tbuf, get_debug_by_level(entry->level), entry->message);
+
+ data.dsize -= size;
+ data.dptr += size;
+ }
+ exit(0);
+}
+
+/*
+ display a list of log messages from the in memory ringbuffer
+ */
+static int control_getlog(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+ int ret;
+ int32_t res;
+ struct ctdb_get_log_addr log_addr;
+ TDB_DATA data;
+ TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+ char *errmsg;
+ struct timeval tv;
+
+ if (argc != 1) {
+ DEBUG(DEBUG_ERR,("Invalid arguments\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ log_addr.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
+ log_addr.srvid = getpid();
+ if (isalpha(argv[0][0]) || argv[0][0] == '-') {
+ log_addr.level = get_debug_by_desc(argv[0]);
+ } else {
+ log_addr.level = strtol(argv[0], NULL, 0);
+ }
+
+
+ data.dptr = (unsigned char *)&log_addr;
+ data.dsize = sizeof(log_addr);
+
+ DEBUG(DEBUG_ERR, ("Pulling logs from node %u\n", options.pnn));
+
+ ctdb_set_message_handler(ctdb, log_addr.srvid, log_handler, NULL);
+ sleep(1);
+
+ DEBUG(DEBUG_ERR,("Listen for response on %d\n", (int)log_addr.srvid));
+
+ ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_GET_LOG,
+ 0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
+ if (ret != 0 || res != 0) {
+ DEBUG(DEBUG_ERR,("Failed to get logs - %s\n", errmsg));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+
+ tv = timeval_current();
+ /* this loop will terminate when we have received the reply */
+ while (timeval_elapsed(&tv) < 3.0) {
+ event_loop_once(ctdb->ev);
+ }
+
+ DEBUG(DEBUG_INFO,("Timed out waiting for log data.\n"));
+
+ talloc_free(tmp_ctx);
+ return 0;
+}
+
+/*
+ clear the in memory log area
+ */
+static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+ int ret;
+ int32_t res;
+ char *errmsg;
+ TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+
+ ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_CLEAR_LOG,
+ 0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
+ if (ret != 0 || res != 0) {
+ DEBUG(DEBUG_ERR,("Failed to clear logs\n"));
+ talloc_free(tmp_ctx);
+ return -1;
+ }
+
+ talloc_free(tmp_ctx);
+ return 0;
+}
+
+
+
/*
display a list of the databases on a remote ctdb
*/
@@ -3460,6 +3569,8 @@ static const struct {
{ "enablemonitor", control_enable_monmode, true, false, "set monitoring mode to ACTIVE" },
{ "setdebug", control_setdebug, true, false, "set debug level", "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
{ "getdebug", control_getdebug, true, false, "get debug level" },
+ { "getlog", control_getlog, true, false, "get the log data from the in memory ringbuffer", "<level>" },
+ { "clearlog", control_clearlog, true, false, "clear the log data from the in memory ringbuffer" },
{ "attach", control_attach, true, false, "attach to a database", "<dbname>" },
{ "dumpmemory", control_dumpmemory, true, false, "dump memory map to stdout" },
{ "rddumpmemory", control_rddumpmemory, true, false, "dump memory map from the recovery daemon to stdout" },
--
CTDB repository
More information about the samba-cvs
mailing list