[SCM] CTDB repository - branch 1.13 updated - ctdb-1.43-13-g0cd522f

Ronnie Sahlberg sahlberg at samba.org
Wed Jun 13 21:17:17 MDT 2012


The branch, 1.13 has been updated
       via  0cd522f854bb788317e15e5f9a562bdb5abcfb17 (commit)
      from  ba94cccca0a9ccad7c1de0939e74f0163ae41102 (commit)

http://gitweb.samba.org/?p=ctdb.git;a=shortlog;h=1.13


- Log -----------------------------------------------------------------
commit 0cd522f854bb788317e15e5f9a562bdb5abcfb17
Author: Ronnie Sahlberg <ronniesahlberg at gmail.com>
Date:   Wed Jun 13 16:17:18 2012 +1000

    STATISTICS: Add tracking of the 10 hottest keys per database measured in hopcount
    
    and add mechanisms to dump it using the ctdb dbstatistics command

-----------------------------------------------------------------------

Summary of changes:
 include/ctdb_protocol.h   |   14 ++++++++++++
 libctdb/control.c         |   40 ++++++++++++++++++++++++++++++++---
 server/ctdb_call.c        |   50 ++++++++++++++++++++++++++++++++++++++++++++-
 server/ctdb_ltdb_server.c |   38 ++++++++++++++++++++++++++++++++-
 tools/ctdb.c              |    9 ++++++++
 5 files changed, 144 insertions(+), 7 deletions(-)


Changeset truncated at 500 lines:

diff --git a/include/ctdb_protocol.h b/include/ctdb_protocol.h
index 5c787ff..33187c7 100644
--- a/include/ctdb_protocol.h
+++ b/include/ctdb_protocol.h
@@ -614,6 +614,7 @@ struct ctdb_traverse_start_ext {
   ctdb statistics information
  */
 #define MAX_COUNT_BUCKETS 16
+#define MAX_HOT_KEYS      10
 
 struct ctdb_statistics {
 	uint32_t num_clients;
@@ -680,10 +681,23 @@ struct ctdb_statistics_wire {
 /*
  * db statistics
  */
+struct ctdb_db_hot_key {
+	uint32_t count;
+	TDB_DATA key;
+};
 struct ctdb_db_statistics {
 	uint32_t db_ro_delegations;
 	uint32_t db_ro_revokes;
 	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+	uint32_t num_hot_keys;
+	struct ctdb_db_hot_key hot_keys[MAX_HOT_KEYS];
+};
+struct ctdb_db_statistics_wire {
+	uint32_t db_ro_delegations;
+	uint32_t db_ro_revokes;
+	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+	uint32_t num_hot_keys;
+	char hot_keys[1];
 };
 
 /*
diff --git a/libctdb/control.c b/libctdb/control.c
index b4c54cd..f927e08 100644
--- a/libctdb/control.c
+++ b/libctdb/control.c
@@ -120,6 +120,9 @@ bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
 {
 	struct ctdb_reply_control *reply;
 	struct ctdb_db_statistics *s;
+	struct ctdb_db_statistics_wire *wire;
+	int i;
+	char *ptr;
 
 	reply = unpack_reply_control(req, CTDB_CONTROL_GET_DB_STATISTICS);
 	if (!reply) {
@@ -129,16 +132,36 @@ bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
 		DEBUG(ctdb, LOG_ERR, "ctdb_getpnn_recv: status -1");
 		return false;
 	}
-	if (reply->datalen != sizeof(struct ctdb_db_statistics)) {
-		DEBUG(ctdb, LOG_ERR, "ctdb_getdbstat_recv: returned data is %d bytes but should be %d", reply->datalen, (int)sizeof(struct ctdb_db_statistics));
+	if (reply->datalen < offsetof(struct ctdb_db_statistics_wire, hot_keys)) {
+		DEBUG(ctdb, LOG_ERR, "ctdb_getdbstat_recv: returned data is %d bytes but should be >= %d", reply->datalen, (int)sizeof(struct ctdb_db_statistics));
 		return false;
 	}
 
-	s = malloc(sizeof(struct ctdb_db_statistics));
+	wire = reply->data;
+
+	s = malloc(offsetof(struct ctdb_db_statistics, hot_keys) + sizeof(struct ctdb_db_hot_key) * wire->num_hot_keys);
 	if (!s) {
 		return false;
 	}
-	memcpy(s, reply->data, sizeof(struct ctdb_db_statistics));
+	s->db_ro_delegations = wire->db_ro_delegations;
+	s->db_ro_revokes     = wire->db_ro_revokes;
+	for (i = 0; i < MAX_COUNT_BUCKETS; i++) {
+		s->hop_count_bucket[i] = wire->hop_count_bucket[i];
+	}
+	s->num_hot_keys      = wire->num_hot_keys;
+	ptr = &wire->hot_keys[0];
+	for (i = 0; i < wire->num_hot_keys; i++) {
+		s->hot_keys[i].count = *(uint32_t *)ptr;
+		ptr += 4;
+
+		s->hot_keys[i].key.dsize = *(uint32_t *)ptr;
+		ptr += 4;
+
+		s->hot_keys[i].key.dptr = malloc(s->hot_keys[i].key.dsize);
+		memcpy(s->hot_keys[i].key.dptr, ptr, s->hot_keys[i].key.dsize);
+		ptr += s->hot_keys[i].key.dsize;
+	}
+
 	*stat = s;
 
 	return true;
@@ -158,9 +181,18 @@ struct ctdb_request *ctdb_getdbstat_send(struct ctdb_connection *ctdb,
 
 void ctdb_free_dbstat(struct ctdb_db_statistics *stat)
 {
+	int i;
+
 	if (stat == NULL) {
 		return;
 	}
+
+	for (i = 0; i < stat->num_hot_keys; i++) {
+		if (stat->hot_keys[i].key.dptr != NULL) {
+			free(stat->hot_keys[i].key.dptr);
+		}
+	}
+
 	free(stat);
 }
 
diff --git a/server/ctdb_call.c b/server/ctdb_call.c
index fe7e947..56cb5e8 100644
--- a/server/ctdb_call.c
+++ b/server/ctdb_call.c
@@ -667,6 +667,54 @@ ctdb_defer_pinned_down_request(struct ctdb_context *ctdb, struct ctdb_db_context
 	return 0;
 }
 
+static void
+ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int hopcount)
+{
+	int i;
+
+	/* smallest value is always at index 0 */
+	if (hopcount <= ctdb_db->statistics.hot_keys[0].count) {
+		return;
+	}
+
+	/* see if we already know this key */
+	for (i = 0; i < MAX_HOT_KEYS; i++) {
+		if (key.dsize != ctdb_db->statistics.hot_keys[i].key.dsize) {
+			continue;
+		}
+		if (memcmp(key.dptr, ctdb_db->statistics.hot_keys[i].key.dptr, key.dsize)) {
+			continue;
+		}
+		/* found an entry for this key */
+		if (hopcount <= ctdb_db->statistics.hot_keys[i].count) {
+			return;
+		}
+		ctdb_db->statistics.hot_keys[i].count = hopcount;
+		goto sort_keys;
+	}
+
+	if (ctdb_db->statistics.hot_keys[0].key.dptr != NULL) {
+		talloc_free(ctdb_db->statistics.hot_keys[0].key.dptr);
+	}
+	ctdb_db->statistics.hot_keys[0].key.dsize = key.dsize;
+	ctdb_db->statistics.hot_keys[0].key.dptr  = talloc_memdup(ctdb_db, key.dptr, key.dsize);
+	ctdb_db->statistics.hot_keys[0].count = hopcount;
+
+
+sort_keys:
+	for (i = 2; i < MAX_HOT_KEYS; i++) {
+		if (ctdb_db->statistics.hot_keys[i].count < ctdb_db->statistics.hot_keys[0].count) {
+			hopcount = ctdb_db->statistics.hot_keys[i].count;
+			ctdb_db->statistics.hot_keys[i].count = ctdb_db->statistics.hot_keys[0].count;
+			ctdb_db->statistics.hot_keys[0].count = hopcount;
+
+			key = ctdb_db->statistics.hot_keys[i].key;
+			ctdb_db->statistics.hot_keys[i].key = ctdb_db->statistics.hot_keys[0].key;
+			ctdb_db->statistics.hot_keys[0].key = key;
+		}
+	}
+}
+
 /*
   called when a CTDB_REQ_CALL packet comes in
 */
@@ -867,7 +915,7 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	}
 	CTDB_INCREMENT_STAT(ctdb, hop_count_bucket[bucket]);
 	CTDB_INCREMENT_DB_STAT(ctdb_db, hop_count_bucket[bucket]);
-
+	ctdb_update_db_stat_hot_keys(ctdb_db, call->key, c->hopcount);
 
 	/* If this database supports sticky records, then check if the
 	   hopcount is big. If it is it means the record is hot and we
diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index 1d21c07..0aa378b 100644
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -1493,6 +1493,10 @@ int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
 				TDB_DATA *outdata)
 {
 	struct ctdb_db_context *ctdb_db;
+	struct ctdb_db_statistics_wire *stats;
+	int i;
+	int len;
+	char *ptr;
 
 	ctdb_db = find_ctdb_db(ctdb, db_id);
 	if (!ctdb_db) {
@@ -1500,8 +1504,38 @@ int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
 		return -1;
 	}
 
-	outdata->dptr  = (uint8_t *)&(ctdb_db->statistics);
-	outdata->dsize = sizeof(ctdb_db->statistics);
+	len = offsetof(struct ctdb_db_statistics_wire, hot_keys);
+	for (i = 0; i < MAX_HOT_KEYS; i++) {
+		len += 8 + ctdb_db->statistics.hot_keys[i].key.dsize;
+	}
+
+	stats = talloc_size(outdata, len);
+	if (stats == NULL) {
+		DEBUG(DEBUG_ERR,("Failed to allocate db statistics wire structure\n"));
+		return -1;
+	}
+
+	stats->db_ro_delegations = ctdb_db->statistics.db_ro_delegations;
+	stats->db_ro_revokes     = ctdb_db->statistics.db_ro_revokes;
+	for (i = 0; i < MAX_COUNT_BUCKETS; i++) {
+		stats->hop_count_bucket[i] = ctdb_db->statistics.hop_count_bucket[i];
+	}
+	stats->num_hot_keys = MAX_HOT_KEYS;
+
+	ptr = &stats->hot_keys[0];
+	for (i = 0; i < MAX_HOT_KEYS; i++) {
+		*(uint32_t *)ptr = ctdb_db->statistics.hot_keys[i].count;
+		ptr += 4;
+
+		*(uint32_t *)ptr = ctdb_db->statistics.hot_keys[i].key.dsize;
+		ptr += 4;
+
+		memcpy(ptr, ctdb_db->statistics.hot_keys[i].key.dptr, ctdb_db->statistics.hot_keys[i].key.dsize);
+		ptr += ctdb_db->statistics.hot_keys[i].key.dsize;
+	}
+
+	outdata->dptr  = (uint8_t *)stats;
+	outdata->dsize = len;
 
 	return 0;
 }
diff --git a/tools/ctdb.c b/tools/ctdb.c
index a3bcd6e..07a47db 100644
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -619,6 +619,15 @@ static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char
 		printf(" %d", dbstatistics->hop_count_bucket[i]);
 	}
 	printf("\n");
+	printf("Num Hot Keys:     %d\n", dbstatistics->num_hot_keys);
+	for (i = 0; i < dbstatistics->num_hot_keys; i++) {
+		int j;
+		printf("Count:%d Key:", dbstatistics->hot_keys[i].count);
+		for (j = 0; j < dbstatistics->hot_keys[i].key.dsize; j++) {
+			printf("%02x", dbstatistics->hot_keys[i].key.dptr[j]&0xff);
+		}
+		printf("\n");
+	}
 
 	ctdb_free_dbstat(dbstatistics);
 	return 0;


-- 
CTDB repository


More information about the samba-cvs mailing list