[SCM] Samba Shared Repository - branch master updated
Martin Schwenke
martins at samba.org
Wed Apr 5 06:36:02 UTC 2017
The branch, master has been updated
via 5da4719 ctdb-daemon: Add tracking of migration records
via 1445fa3 ctdb-daemon: For hot records, use count instead of hopcount
via 1b2c919 ctdb-common: Add hash_count abstraction
via 63b92f4 ctdb-common: Add traverse_update function to db_hash abstraction
from 9566eb2 TestBase: restore setting FEATURE_SEAL in insta_creds
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 5da471919d2b7ed45be574a8cb7c88351f894797
Author: Amitay Isaacs <amitay at gmail.com>
Date: Tue Mar 21 16:48:45 2017 +1100
ctdb-daemon: Add tracking of migration records
Instead of using hopcount as a metric for hot records, use the number
of migrations per second as a metric.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
Autobuild-User(master): Martin Schwenke <martins at samba.org>
Autobuild-Date(master): Wed Apr 5 08:35:45 CEST 2017 on sn-devel-144
commit 1445fa3b7126a3e12fc795db2d7a495f9caf2a3a
Author: Amitay Isaacs <amitay at gmail.com>
Date: Mon Apr 3 17:32:32 2017 +1000
ctdb-daemon: For hot records, use count instead of hopcount
This avoids tying hopcounts to hot records.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit 1b2c919f1956175a78693611908da13f11ce024a
Author: Amitay Isaacs <amitay at gmail.com>
Date: Fri Mar 17 18:00:40 2017 +1100
ctdb-common: Add hash_count abstraction
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit 63b92f42f6f525c6fab97bf37cf52587b7458a76
Author: Amitay Isaacs <amitay at gmail.com>
Date: Fri Mar 17 18:00:16 2017 +1100
ctdb-common: Add traverse_update function to db_hash abstraction
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
-----------------------------------------------------------------------
Summary of changes:
ctdb/common/db_hash.c | 27 +++
ctdb/common/db_hash.h | 19 +-
ctdb/common/hash_count.c | 219 +++++++++++++++++++++
ctdb/common/hash_count.h | 94 +++++++++
ctdb/include/ctdb_private.h | 4 +
ctdb/server/ctdb_call.c | 94 ++++++++-
ctdb/server/ctdb_ltdb_server.c | 9 +
.../{srvid_test_001.sh => hash_count_test_001.sh} | 2 +-
ctdb/tests/src/hash_count_test.c | 132 +++++++++++++
ctdb/wscript | 4 +-
10 files changed, 590 insertions(+), 14 deletions(-)
create mode 100644 ctdb/common/hash_count.c
create mode 100644 ctdb/common/hash_count.h
copy ctdb/tests/cunit/{srvid_test_001.sh => hash_count_test_001.sh} (67%)
create mode 100644 ctdb/tests/src/hash_count_test.c
Changeset truncated at 500 lines:
diff --git a/ctdb/common/db_hash.c b/ctdb/common/db_hash.c
index 6a23337..8dd62c4 100644
--- a/ctdb/common/db_hash.c
+++ b/ctdb/common/db_hash.c
@@ -266,3 +266,30 @@ int db_hash_traverse(struct db_hash_context *dh,
return ret;
}
+
+int db_hash_traverse_update(struct db_hash_context *dh,
+ db_hash_record_parser_fn parser,
+ void *private_data, int *count)
+{
+ struct db_hash_traverse_state state;
+ int ret;
+
+ if (dh == NULL || parser == NULL) {
+ return EINVAL;
+ }
+
+ state.parser = parser;
+ state.private_data = private_data;
+
+ ret = tdb_traverse(dh->db, db_hash_traverse_parser, &state);
+ if (ret == -1) {
+ ret = db_hash_map_tdb_error(dh);
+ } else {
+ if (count != NULL) {
+ *count = ret;
+ }
+ ret = 0;
+ }
+
+ return ret;
+}
diff --git a/ctdb/common/db_hash.h b/ctdb/common/db_hash.h
index 8f0e50b..67e2b85 100644
--- a/ctdb/common/db_hash.h
+++ b/ctdb/common/db_hash.h
@@ -142,9 +142,9 @@ int db_hash_fetch(struct db_hash_context *dh, uint8_t *keybuf, size_t keylen,
int db_hash_exists(struct db_hash_context *dh, uint8_t *keybuf, size_t keylen);
/**
- * @brief Traverse the database
+ * @brief Traverse the database without modification
*
- * The parser function should non-zero value to stop traverse.
+ * The parser function should return non-zero value to stop traverse.
*
* @param[in] dh The tdb hash table context
* @param[in] parser Function called for each record
@@ -156,4 +156,19 @@ int db_hash_traverse(struct db_hash_context *dh,
db_hash_record_parser_fn parser, void *private_data,
int *count);
+/**
+ * @brief Traverse the database for modifications
+ *
+ * The parser function should return non-zero value to stop traverse.
+ *
+ * @param[in] dh The tdb hash table context
+ * @param[in] parser Function called for each record
+ * @param[in] private_data Private data to parser function
+ * @param[out] count Number of records traversed
+ * @return 0 on success, errno on failure
+ */
+int db_hash_traverse_update(struct db_hash_context *dh,
+ db_hash_record_parser_fn parser,
+ void *private_data, int *count);
+
#endif /* __CTDB_DB_HASH_H__ */
diff --git a/ctdb/common/hash_count.c b/ctdb/common/hash_count.c
new file mode 100644
index 0000000..f845016
--- /dev/null
+++ b/ctdb/common/hash_count.c
@@ -0,0 +1,219 @@
+/*
+ Using hash table for counting events
+
+ Copyright (C) Amitay Isaacs 2017
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "replace.h"
+#include "system/filesys.h"
+#include "system/time.h"
+
+#include <tdb.h>
+
+#include "lib/util/time.h"
+
+#include "common/db_hash.h"
+#include "common/hash_count.h"
+
+struct hash_count_value {
+ struct timeval update_time;
+ uint64_t counter;
+};
+
+struct hash_count_context {
+ struct db_hash_context *dh;
+ struct timeval update_interval;
+ hash_count_update_handler_fn handler;
+ void *private_data;
+};
+
+/*
+ * Initialise hash count map
+ */
+int hash_count_init(TALLOC_CTX *mem_ctx, struct timeval update_interval,
+ hash_count_update_handler_fn handler, void *private_data,
+ struct hash_count_context **result)
+{
+ struct hash_count_context *hcount;
+ int ret;
+
+ if (handler == NULL) {
+ return EINVAL;
+ }
+
+ hcount = talloc_zero(mem_ctx, struct hash_count_context);
+ if (hcount == NULL) {
+ return ENOMEM;
+ }
+
+ ret = db_hash_init(hcount, "hash_count_db", 8192, DB_HASH_COMPLEX,
+ &hcount->dh);
+ if (ret != 0) {
+ talloc_free(hcount);
+ return ret;
+ }
+
+ hcount->update_interval = update_interval;
+ hcount->handler = handler;
+ hcount->private_data = private_data;
+
+ *result = hcount;
+ return 0;
+}
+
+static int hash_count_fetch_parser(uint8_t *keybuf, size_t keylen,
+ uint8_t *databuf, size_t datalen,
+ void *private_data)
+{
+ struct hash_count_value *value =
+ (struct hash_count_value *)private_data;
+
+ if (datalen != sizeof(struct hash_count_value)) {
+ return EIO;
+ }
+
+ *value = *(struct hash_count_value *)databuf;
+ return 0;
+}
+
+static int hash_count_fetch(struct hash_count_context *hcount, TDB_DATA key,
+ struct hash_count_value *value)
+{
+ return db_hash_fetch(hcount->dh, key.dptr, key.dsize,
+ hash_count_fetch_parser, value);
+}
+
+static int hash_count_insert(struct hash_count_context *hcount, TDB_DATA key,
+ struct hash_count_value *value)
+{
+ return db_hash_insert(hcount->dh, key.dptr, key.dsize,
+ (uint8_t *)value,
+ sizeof(struct hash_count_value));
+}
+
+static int hash_count_update(struct hash_count_context *hcount, TDB_DATA key,
+ struct hash_count_value *value)
+{
+ return db_hash_add(hcount->dh, key.dptr, key.dsize,
+ (uint8_t *)value, sizeof(struct hash_count_value));
+}
+
+int hash_count_increment(struct hash_count_context *hcount, TDB_DATA key)
+{
+ struct hash_count_value value;
+ struct timeval current_time = timeval_current();
+ int ret;
+
+ if (hcount == NULL) {
+ return EINVAL;
+ }
+
+ ret = hash_count_fetch(hcount, key, &value);
+ if (ret == 0) {
+ struct timeval tmp_t;
+
+ tmp_t = timeval_sum(&value.update_time,
+ &hcount->update_interval);
+ if (timeval_compare(¤t_time, &tmp_t) < 0) {
+ value.counter += 1;
+ } else {
+ value.update_time = current_time;
+ value.counter = 1;
+ }
+
+ hcount->handler(key, value.counter, hcount->private_data);
+ ret = hash_count_update(hcount, key, &value);
+
+ } else if (ret == ENOENT) {
+ value.update_time = current_time;
+ value.counter = 1;
+
+ hcount->handler(key, value.counter, hcount->private_data);
+ ret = hash_count_insert(hcount, key, &value);
+ }
+
+ return ret;
+}
+
+static struct timeval timeval_subtract(const struct timeval *tv1,
+ const struct timeval *tv2)
+{
+ struct timeval tv = *tv1;
+ const unsigned int million = 1000000;
+
+ if (tv.tv_sec > 1) {
+ tv.tv_sec -= 1;
+ tv.tv_usec += million;
+ } else {
+ return tv;
+ }
+
+ tv.tv_sec -= tv2->tv_sec;
+ tv.tv_usec -= tv2->tv_usec;
+
+ tv.tv_sec += tv.tv_usec / million;
+ tv.tv_usec = tv.tv_usec % million;
+
+ return tv;
+}
+
+struct hash_count_expire_state {
+ struct db_hash_context *dh;
+ struct timeval last_time;
+ int count;
+};
+
+static int hash_count_expire_parser(uint8_t *keybuf, size_t keylen,
+ uint8_t *databuf, size_t datalen,
+ void *private_data)
+{
+ struct hash_count_expire_state *state =
+ (struct hash_count_expire_state *)private_data;
+ struct hash_count_value *value;
+ int ret = 0;
+
+ if (datalen != sizeof(struct hash_count_value)) {
+ return EIO;
+ }
+
+ value = (struct hash_count_value *)databuf;
+ if (timeval_compare(&value->update_time, &state->last_time) < 0) {
+ ret = db_hash_delete(state->dh, keybuf, keylen);
+ if (ret == 0) {
+ state->count += 1;
+ }
+ }
+
+ return ret;
+}
+
+void hash_count_expire(struct hash_count_context *hcount, int *delete_count)
+{
+ struct timeval current_time = timeval_current();
+ struct hash_count_expire_state state;
+
+ state.dh = hcount->dh;
+ state.last_time = timeval_subtract(¤t_time,
+ &hcount->update_interval);
+ state.count = 0;
+
+ (void) db_hash_traverse_update(hcount->dh, hash_count_expire_parser,
+ &state, NULL);
+
+ if (delete_count != NULL) {
+ *delete_count = state.count;
+ }
+}
diff --git a/ctdb/common/hash_count.h b/ctdb/common/hash_count.h
new file mode 100644
index 0000000..f14c82c
--- /dev/null
+++ b/ctdb/common/hash_count.h
@@ -0,0 +1,94 @@
+/*
+ Using hash table for counting events
+
+ Copyright (C) Amitay Isaacs 2017
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __CTDB_HASH_COUNT_H__
+#define __CTDB_HASH_COUNT_H__
+
+/**
+ * @file hash_count.h
+ *
+ * @brief Count key-based events for specified interval
+ *
+ * This can be used to measure the rate of events based on any interval.
+ * For example, number of occurrences per second.
+ */
+
+/**
+ * @brief Handler callback function called when counter is incremented
+ *
+ * This function is called every time a counter is incremented for a key.
+ * The counter argument is the number of times the increment function is
+ * called during a count interval.
+ *
+ * This function should not modify key and data arguments.
+ */
+typedef void (*hash_count_update_handler_fn)(TDB_DATA key, uint64_t counter,
+ void *private_data);
+
+/**
+ * @brief Abstract structure representing hash based counting
+ */
+struct hash_count_context;
+
+/**
+ * @brief Initialize hash counting
+ *
+ * This return a new hash count context which is a talloc context. Freeing
+ * this context will free all the memory associated with hash count.
+ *
+ * @param[in] mem_ctx Talloc memory context
+ * @param[in] count_interval The time interval for counting events
+ * @param[in] handler Function called when counter is incremented
+ * @param[in] private_data Private data to handler function
+ * @param[out] result The new hash_count structure
+ * @return 0 on success, errno on failure
+ */
+int hash_count_init(TALLOC_CTX *mem_ctx, struct timeval count_interval,
+ hash_count_update_handler_fn handler, void *private_data,
+ struct hash_count_context **result);
+
+/**
+ * @brief Increment a counter for a key
+ *
+ * First time this is called for a key, corresponding counter is set to 1
+ * and the start time is noted. For all subsequent calls made during the
+ * count_interval (used in initializing the context) will increment
+ * corresponding counter for the key. After the count_interval has elapsed,
+ * the counter will be reset to 1.
+ *
+ * @param[in] hcount The hash count context
+ * @param[in] key The key for which counter is updated
+ * @return 0 on success, errno on failure
+ *
+ * This will result in a callback function being called.
+ */
+int hash_count_increment(struct hash_count_context *hcount, TDB_DATA key);
+
+/**
+ * @brief Remove keys for which count interval has elapsed
+ *
+ * This function is used to clean the database of keys for which there are
+ * no recent events.
+ *
+ * @param[in] hcount The hash count context
+ * @param[out] delete_count The number of keys deleted
+ */
+void hash_count_expire(struct hash_count_context *hcount, int *delete_count);
+
+#endif /* __CTDB_HASH_COUNT_H__ */
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index d81ed56..91e9885 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -387,6 +387,8 @@ struct ctdb_db_context {
bool push_started;
void *push_state;
+
+ struct hash_count_context *migratedb;
};
@@ -521,6 +523,8 @@ int ctdb_add_revoke_deferred_call(struct ctdb_context *ctdb,
TDB_DATA key, struct ctdb_req_header *hdr,
deferred_requeue_fn fn, void *call_context);
+int ctdb_migration_init(struct ctdb_db_context *ctdb_db);
+
/* from server/ctdb_control.c */
int32_t ctdb_dump_memory(struct ctdb_context *ctdb, TDB_DATA *outdata);
diff --git a/ctdb/server/ctdb_call.c b/ctdb/server/ctdb_call.c
index 8ce3928..ed943f9 100644
--- a/ctdb/server/ctdb_call.c
+++ b/ctdb/server/ctdb_call.c
@@ -41,6 +41,7 @@
#include "common/system.h"
#include "common/common.h"
#include "common/logging.h"
+#include "common/hash_count.h"
struct ctdb_sticky_record {
struct ctdb_context *ctdb;
@@ -416,6 +417,8 @@ static void ctdb_become_dmaster(struct ctdb_db_context *ctdb_db,
return;
}
+ (void) hash_count_increment(ctdb_db->migratedb, key);
+
ctdb_call_local(ctdb_db, state->call, &header, state, &data, true);
ret = ctdb_ltdb_unlock(ctdb_db, state->call->key);
@@ -818,12 +821,13 @@ ctdb_defer_pinned_down_request(struct ctdb_context *ctdb, struct ctdb_db_context
}
static void
-ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int hopcount)
+ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key,
+ int count)
{
int i, id;
/* smallest value is always at index 0 */
- if (hopcount <= ctdb_db->statistics.hot_keys[0].count) {
+ if (count <= ctdb_db->statistics.hot_keys[0].count) {
return;
}
@@ -836,10 +840,10 @@ ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int
continue;
}
/* found an entry for this key */
- if (hopcount <= ctdb_db->statistics.hot_keys[i].count) {
+ if (count <= ctdb_db->statistics.hot_keys[i].count) {
return;
}
- ctdb_db->statistics.hot_keys[i].count = hopcount;
+ ctdb_db->statistics.hot_keys[i].count = count;
goto sort_keys;
}
@@ -855,9 +859,10 @@ ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int
}
ctdb_db->statistics.hot_keys[id].key.dsize = key.dsize;
ctdb_db->statistics.hot_keys[id].key.dptr = talloc_memdup(ctdb_db, key.dptr, key.dsize);
- ctdb_db->statistics.hot_keys[id].count = hopcount;
- DEBUG(DEBUG_NOTICE,("Updated hot key database=%s key=0x%08x id=%d hop_count=%d\n",
- ctdb_db->db_name, ctdb_hash(&key), id, hopcount));
+ ctdb_db->statistics.hot_keys[id].count = count;
+ DEBUG(DEBUG_NOTICE,
+ ("Updated hot key database=%s key=0x%08x id=%d count=%d\n",
+ ctdb_db->db_name, ctdb_hash(&key), id, count));
sort_keys:
for (i = 1; i < MAX_HOT_KEYS; i++) {
@@ -865,9 +870,9 @@ sort_keys:
continue;
}
if (ctdb_db->statistics.hot_keys[i].count < ctdb_db->statistics.hot_keys[0].count) {
- hopcount = ctdb_db->statistics.hot_keys[i].count;
+ count = ctdb_db->statistics.hot_keys[i].count;
ctdb_db->statistics.hot_keys[i].count = ctdb_db->statistics.hot_keys[0].count;
- ctdb_db->statistics.hot_keys[0].count = hopcount;
+ ctdb_db->statistics.hot_keys[0].count = count;
key = ctdb_db->statistics.hot_keys[i].key;
ctdb_db->statistics.hot_keys[i].key = ctdb_db->statistics.hot_keys[0].key;
@@ -1090,7 +1095,6 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
}
CTDB_INCREMENT_STAT(ctdb, hop_count_bucket[bucket]);
CTDB_INCREMENT_DB_STAT(ctdb_db, hop_count_bucket[bucket]);
- ctdb_update_db_stat_hot_keys(ctdb_db, call->key, c->hopcount);
--
Samba Shared Repository
More information about the samba-cvs
mailing list