Status of 64-bit TDB

[ronnie sahlberg]

Fragmentation of TDB databases is a big problem in CTDB.
For very busy systems it usually does not take that long until the databases
are so fragmented
they become very slow.

For this reason ctdb has a commandline utility "ctdb repack" that does this
kind defragmentation.

Please see the ctdb git tree : tools/ctdb_vacuum.c:ctdb_repack_tdb for an
example how this is done in CTDB.

Example code:


struct traverse_state {
    bool error;
    struct tdb_context *dest_db;
};

/*
  traverse function for repacking
 */
static int repack_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA
data, void *private)
{
    struct traverse_state *state = (struct traverse_state *)private;
    if (tdb_store(state->dest_db, key, data, TDB_INSERT) != 0) {
        state->error = true;
        return -1;
    }
    return 0;
}

/*
  repack a tdb
 */
static int ctdb_repack_tdb(struct tdb_context *tdb)
{
    struct tdb_context *tmp_db;
    struct traverse_state state;

    if (tdb_transaction_start(tdb) != 0) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction\n"));
        return -1;
    }

    tmp_db = tdb_open("tmpdb", tdb_hash_size(tdb), TDB_INTERNAL,
O_RDWR|O_CREAT, 0);
    if (tmp_db == NULL) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to create tmp_db\n"));
        tdb_transaction_cancel(tdb);
        return -1;
    }
    state.error = false;
    state.dest_db = tmp_db;

    if (tdb_traverse_read(tdb, repack_traverse, &state) == -1) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to traverse copying out\n"));
        tdb_transaction_cancel(tdb);
        tdb_close(tmp_db);
        return -1;
    }

    if (state.error) {
        DEBUG(DEBUG_ERR,(__location__ " Error during traversal\n"));
        tdb_transaction_cancel(tdb);
        tdb_close(tmp_db);
        return -1;
    }

    if (tdb_wipe_all(tdb) != 0) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to wipe database\n"));
        tdb_transaction_cancel(tdb);
        tdb_close(tmp_db);
        return -1;
    }

    state.error = false;
    state.dest_db = tdb;

    if (tdb_traverse_read(tmp_db, repack_traverse, &state) == -1) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to traverse copying
back\n"));
        tdb_transaction_cancel(tdb);
        tdb_close(tmp_db);
        return -1;
    }

    if (state.error) {
        DEBUG(DEBUG_ERR,(__location__ " Error during second traversal\n"));
        tdb_transaction_cancel(tdb);
        tdb_close(tmp_db);
        return -1;
    }

    tdb_close(tmp_db);

    if (tdb_transaction_commit(tdb) != 0) {
        DEBUG(DEBUG_ERR,(__location__ " Failed to commit\n"));
        return -1;
    }

    return 0;
}






On Tue, Jun 16, 2009 at 11:35 PM, Volker Lendecke <Volker.Lendecke at sernet.de
> wrote:

> On Tue, Jun 16, 2009 at 09:21:36AM -0400, yaberger at ca.ibm.com wrote:
> > this might not be the solution for John's problem but I think a TDB
> > defragmenter/shrinker is something that is missing around TDBs.
> >
> > Currently, the only way to reduce the size of a TDB is to stop Samba and
> > delete the non-persistant TDBs.
>
> You can always do a tdbbackup of the persistent ones.
>
> > Has it been discussed in the past?
> > Is this something that makes sense and would be doable?
>
> Sure. It will need to be coded up very carefully though if
> it shall work while the tdb is in use.
>
> Volker
>