[SCM] Samba Shared Repository - branch master updated
Amitay Isaacs
amitay at samba.org
Fri Dec 5 09:07:02 MST 2014
The branch, master has been updated
via 959b9ea ctdb-recoverd: Process all the records for vacuum fetch in a loop
via 257311e ctdb-vacuum: Do not delete VACUUM MIGRATED records immediately
via dbb1958 ctdb-vacuum: Use non-blocking lock when traversing delete tree
via d35f512 ctdb-vacuum: Use non-blocking lock when traversing delete queue
via e4597f8 ctdb-vacuum: Stagger vacuuming child processes
via a0628e3 ctdb-vacuum: Track time for vacuuming in database statistics
from 8f4813f vfs_fruit: add AAPL options
https://git.samba.org/?p=samba.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 959b9ea0ef85c57ffc84d66a6e5e855868943391
Author: Amitay Isaacs <amitay at gmail.com>
Date: Wed Apr 23 18:02:39 2014 +1000
ctdb-recoverd: Process all the records for vacuum fetch in a loop
Processing one migration request at a time is very slow and processing
a batch of records can take longer than VacuumInterval. This causes
subsequent vacuum fetch requests to be dropped. The dropped records
can accumulate quickly and will cause the vacuum database traverse to
be quite expensive.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
Autobuild-User(master): Amitay Isaacs <amitay at samba.org>
Autobuild-Date(master): Fri Dec 5 17:06:58 CET 2014 on sn-devel-104
commit 257311e337065f089df688cbf261d2577949203d
Author: Amitay Isaacs <amitay at gmail.com>
Date: Mon Apr 14 14:53:25 2014 +1000
ctdb-vacuum: Do not delete VACUUM MIGRATED records immediately
Such records should be processed by the local vacuuming daemon to ensure
that all the remote copies have been deleted first.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit dbb1958284657f26a868705e5f9612bc377fd5e0
Author: Amitay Isaacs <amitay at gmail.com>
Date: Thu Nov 6 09:33:50 2014 +1100
ctdb-vacuum: Use non-blocking lock when traversing delete tree
This avoids vacuuming getting in the way of ctdb daemon to process
record requests.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit d35f512cd972ac1f732fe998b2179242d042082d
Author: Amitay Isaacs <amitay at gmail.com>
Date: Mon Apr 14 13:18:41 2014 +1000
ctdb-vacuum: Use non-blocking lock when traversing delete queue
This avoids vacuuming getting in the way of ctdb daemon to process
record requests.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit e4597f8771f42cf315bd163c18b2f27147d3de5f
Author: Amitay Isaacs <amitay at gmail.com>
Date: Fri Feb 21 14:58:00 2014 +1100
ctdb-vacuum: Stagger vacuuming child processes
This prevents multiple child processes being forked at the same time
for vacuuming TDBs.
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
commit a0628e317df76c7c38a7cca9c3090077fa352899
Author: Amitay Isaacs <amitay at gmail.com>
Date: Tue Feb 11 14:23:28 2014 +1100
ctdb-vacuum: Track time for vacuuming in database statistics
Signed-off-by: Amitay Isaacs <amitay at gmail.com>
Reviewed-by: Martin Schwenke <martin at meltin.net>
-----------------------------------------------------------------------
Summary of changes:
ctdb/include/ctdb_protocol.h | 3 +++
ctdb/server/ctdb_ltdb_server.c | 5 +++++
ctdb/server/ctdb_recoverd.c | 5 +----
ctdb/server/ctdb_vacuum.c | 24 ++++++++++++++----------
ctdb/tools/ctdb.c | 8 ++++++++
5 files changed, 31 insertions(+), 14 deletions(-)
Changeset truncated at 500 lines:
diff --git a/ctdb/include/ctdb_protocol.h b/ctdb/include/ctdb_protocol.h
index 725e426..15d9fc5 100644
--- a/ctdb/include/ctdb_protocol.h
+++ b/ctdb/include/ctdb_protocol.h
@@ -717,6 +717,9 @@ struct ctdb_db_statistics {
struct latency_counter latency;
uint32_t buckets[MAX_COUNT_BUCKETS];
} locks;
+ struct {
+ struct latency_counter latency;
+ } vacuum;
uint32_t db_ro_delegations;
uint32_t db_ro_revokes;
uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c
index 8fb2bc7..9ac2217 100644
--- a/ctdb/server/ctdb_ltdb_server.c
+++ b/ctdb/server/ctdb_ltdb_server.c
@@ -115,6 +115,11 @@ static int ctdb_ltdb_store_server(struct ctdb_db_context *ctdb_db,
* fails. So storing the empty record makes sure that we do not
* need to change the client code.
*/
+ if ((header->flags & CTDB_REC_FLAG_VACUUM_MIGRATED) &&
+ (ctdb_db->ctdb->pnn == header->dmaster)) {
+ keep = true;
+ schedule_for_deletion = true;
+ }
if (!(header->flags & CTDB_REC_FLAG_VACUUM_MIGRATED)) {
keep = true;
} else if (ctdb_db->ctdb->pnn != header->dmaster) {
diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index fd07e64..f86f57e 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -910,9 +910,7 @@ static void vacuum_fetch_next(struct vacuum_info *v);
*/
static void vacuum_fetch_callback(struct ctdb_client_call_state *state)
{
- struct vacuum_info *v = talloc_get_type(state->async.private_data, struct vacuum_info);
talloc_free(state);
- vacuum_fetch_next(v);
}
@@ -977,8 +975,7 @@ static void vacuum_fetch_next(struct vacuum_info *v)
return;
}
state->async.fn = vacuum_fetch_callback;
- state->async.private_data = v;
- return;
+ state->async.private_data = NULL;
}
talloc_free(v);
diff --git a/ctdb/server/ctdb_vacuum.c b/ctdb/server/ctdb_vacuum.c
index 35f1fe1..d678ff9 100644
--- a/ctdb/server/ctdb_vacuum.c
+++ b/ctdb/server/ctdb_vacuum.c
@@ -317,12 +317,8 @@ static int delete_marshall_traverse_first(void *param, void *data)
uint32_t hash = ctdb_hash(&(dd->key));
int res;
- res = tdb_chainlock(ctdb_db->ltdb->tdb, dd->key);
+ res = tdb_chainlock_nonblock(ctdb_db->ltdb->tdb, dd->key);
if (res != 0) {
- DEBUG(DEBUG_ERR,
- (__location__ " Error getting chainlock on record with "
- "key hash [0x%08x] on database db[%s].\n",
- hash, ctdb_db->db_name));
recs->vdata->count.delete_list.skipped++;
recs->vdata->count.delete_list.left--;
talloc_free(dd);
@@ -446,12 +442,8 @@ static int delete_queue_traverse(void *param, void *data)
vdata->count.delete_queue.total++;
- res = tdb_chainlock(ctdb_db->ltdb->tdb, dd->key);
+ res = tdb_chainlock_nonblock(ctdb_db->ltdb->tdb, dd->key);
if (res != 0) {
- DEBUG(DEBUG_ERR,
- (__location__ " Error getting chainlock on record with "
- "key hash [0x%08x] on database db[%s].\n",
- hash, ctdb_db->db_name));
vdata->count.delete_queue.error++;
return 0;
}
@@ -1364,6 +1356,7 @@ static int vacuum_child_destructor(struct ctdb_vacuum_child_context *child_ctx)
struct ctdb_db_context *ctdb_db = child_ctx->vacuum_handle->ctdb_db;
struct ctdb_context *ctdb = ctdb_db->ctdb;
+ CTDB_UPDATE_DB_LATENCY(ctdb_db, "vacuum", vacuum.latency, l);
DEBUG(DEBUG_INFO,("Vacuuming took %.3f seconds for database %s\n", l, ctdb_db->db_name));
if (child_ctx->child_pid != -1) {
@@ -1450,6 +1443,17 @@ ctdb_vacuum_event(struct event_context *ev, struct timed_event *te,
return;
}
+ /* Do not allow multiple vacuuming child processes to be active at the
+ * same time. If there is vacuuming child process active, delay
+ * new vacuuming event to stagger vacuuming events.
+ */
+ if (ctdb->vacuumers != NULL) {
+ event_add_timed(ctdb->ev, vacuum_handle,
+ timeval_current_ofs(0, 500*1000),
+ ctdb_vacuum_event, vacuum_handle);
+ return;
+ }
+
child_ctx = talloc(vacuum_handle, struct ctdb_vacuum_child_context);
if (child_ctx == NULL) {
DEBUG(DEBUG_CRIT, (__location__ " Failed to allocate child context for vacuuming of %s\n", ctdb_db->db_name));
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index 5d5ce3c..f0c87d9 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -725,6 +725,14 @@ static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char
0.0),
dbstat->locks.latency.max,
dbstat->locks.latency.num);
+ printf(" %-30s %.6f/%.6f/%.6f sec out of %d\n",
+ "vacuum_latency MIN/AVG/MAX",
+ dbstat->vacuum.latency.min,
+ (dbstat->vacuum.latency.num ?
+ dbstat->vacuum.latency.total /dbstat->vacuum.latency.num :
+ 0.0),
+ dbstat->vacuum.latency.max,
+ dbstat->vacuum.latency.num);
num_hot_keys = 0;
for (i=0; i<dbstat->num_hot_keys; i++) {
if (dbstat->hot_keys[i].count > 0) {
--
Samba Shared Repository
More information about the samba-cvs
mailing list