[SCM] Samba Shared Repository - branch v3-2-test updated -
initial-v3-2-unstable-713-g7a911b3
Volker Lendecke
vl at samba.org
Tue Dec 18 08:53:29 GMT 2007
The branch, v3-2-test has been updated
via 7a911b35713538d82001a3c9f34152e293fe1943 (commit)
from 6a7f2a59fc370e226ddacb195059155f28c6c157 (commit)
http://gitweb.samba.org/?p=samba.git;a=shortlog;h=v3-2-test
- Log -----------------------------------------------------------------
commit 7a911b35713538d82001a3c9f34152e293fe1943
Author: Volker Lendecke <vl at samba.org>
Date: Tue Dec 18 09:41:03 2007 +0100
Add a in-memory cache
This is a more general API that caches data with a LRU scheme. See
include/cache.h. No comments yet, I'm still working on it. But Jeremy has given
me a hint in one of his checkins that he would like to make use of this now.
The idea is that we get rid of all our silly little caches and merge them all
into one cache that we can then very easily trim, for example even with a
smbcontrol message if someone decides memory is tight. The main user is the
stat cache, this patch also converts the getwd cache. More caches to come.
-----------------------------------------------------------------------
Summary of changes:
source/Makefile.in | 2 +-
source/include/includes.h | 1 +
source/lib/cache.c | 298 ++++++++++++++++++++++++++++++++++++++++++++
source/param/loadparm.c | 7 +-
source/smbd/mangle_hash2.c | 67 +++-------
source/smbd/server.c | 13 ++
source/smbd/statcache.c | 64 ++++------
source/smbd/vfs.c | 168 +++++++++----------------
source/torture/torture.c | 76 +++++++++++
source/torture/vfstest.c | 13 ++
10 files changed, 505 insertions(+), 204 deletions(-)
create mode 100644 source/lib/cache.c
Changeset truncated at 500 lines:
diff --git a/source/Makefile.in b/source/Makefile.in
index eda3297..81c8330 100644
--- a/source/Makefile.in
+++ b/source/Makefile.in
@@ -284,7 +284,7 @@ TALLOC_OBJ = lib/talloc/talloc.o
LIB_WITHOUT_PROTO_OBJ = $(LIBREPLACE_OBJ) $(SOCKET_WRAPPER_OBJ) $(NSS_WRAPPER_OBJ) $(TALLOC_OBJ) \
lib/messages.o librpc/gen_ndr/ndr_messaging.o lib/messages_local.o \
lib/messages_ctdbd.o lib/packet.o lib/ctdbd_conn.o lib/talloc_stack.o \
- lib/interfaces.o lib/rbtree.o
+ lib/interfaces.o lib/rbtree.o lib/cache.o
LIB_WITH_PROTO_OBJ = $(VERSION_OBJ) lib/charcnv.o lib/debug.o lib/fault.o \
lib/interface.o lib/md4.o \
diff --git a/source/include/includes.h b/source/include/includes.h
index 2245174..a45176a 100644
--- a/source/include/includes.h
+++ b/source/include/includes.h
@@ -719,6 +719,7 @@ typedef char fstring[FSTRING_LEN];
#include "packet.h"
#include "ctdbd_conn.h"
#include "talloc_stack.h"
+#include "cache.h"
/* used in net.c */
struct functable {
diff --git a/source/lib/cache.c b/source/lib/cache.c
new file mode 100644
index 0000000..baf2fe3
--- /dev/null
+++ b/source/lib/cache.c
@@ -0,0 +1,298 @@
+/*
+ Unix SMB/CIFS implementation.
+ In-memory cache
+ Copyright (C) Volker Lendecke 2007
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "cache.h"
+#include "rbtree.h"
+
+struct memcache_element {
+ struct rb_node rb_node;
+ struct memcache_element *prev, *next;
+ size_t keylength, valuelength;
+ uint8 n; /* This is really an enum, but save memory */
+ char data[1]; /* placeholder for offsetof */
+};
+
+struct memcache {
+ struct memcache_element *mru, *lru;
+ struct rb_root tree;
+ size_t size;
+ size_t max_size;
+};
+
+static int memcache_destructor(struct memcache *cache) {
+ struct memcache_element *e, *next;
+
+ for (e = cache->mru; e != NULL; e = next) {
+ next = e->next;
+ SAFE_FREE(e);
+ }
+ return 0;
+}
+
+struct memcache *memcache_init(TALLOC_CTX *mem_ctx, size_t max_size)
+{
+ struct memcache *result;
+
+ result = TALLOC_ZERO_P(mem_ctx, struct memcache);
+ if (result == NULL) {
+ return NULL;
+ }
+ result->max_size = max_size;
+ talloc_set_destructor(result, memcache_destructor);
+ return result;
+}
+
+static struct memcache_element *memcache_node2elem(struct rb_node *node)
+{
+ return (struct memcache_element *)
+ ((char *)node - offsetof(struct memcache_element, rb_node));
+}
+
+static void memcache_element_parse(struct memcache_element *e,
+ DATA_BLOB *key, DATA_BLOB *value)
+{
+ key->data = ((uint8 *)e) + offsetof(struct memcache_element, data);
+ key->length = e->keylength;
+ value->data = key->data + e->keylength;
+ value->length = e->valuelength;
+}
+
+static size_t memcache_element_size(size_t key_length, size_t value_length)
+{
+ return sizeof(struct memcache_element) - 1 + key_length + value_length;
+}
+
+static int memcache_compare(struct memcache_element *e, enum memcache_number n,
+ DATA_BLOB key)
+{
+ DATA_BLOB this_key, this_value;
+
+ if ((int)e->n < (int)n) return -1;
+ if ((int)e->n > (int)n) return 1;
+
+ if (e->keylength < key.length) return -1;
+ if (e->keylength > key.length) return 1;
+
+ memcache_element_parse(e, &this_key, &this_value);
+ return memcmp(this_key.data, key.data, key.length);
+}
+
+static struct memcache_element *memcache_find(
+ struct memcache *cache, enum memcache_number n, DATA_BLOB key)
+{
+ struct rb_node *node;
+
+ node = cache->tree.rb_node;
+
+ while (node != NULL) {
+ struct memcache_element *elem = memcache_node2elem(node);
+ int cmp;
+
+ cmp = memcache_compare(elem, n, key);
+ if (cmp == 0) {
+ return elem;
+ }
+ node = (cmp < 0) ? node->rb_left : node->rb_right;
+ }
+
+ return NULL;
+}
+
+bool memcache_lookup(struct memcache *cache, enum memcache_number n,
+ DATA_BLOB key, DATA_BLOB *value)
+{
+ struct memcache_element *e;
+
+ e = memcache_find(cache, n, key);
+ if (e == NULL) {
+ return false;
+ }
+
+ if (cache->size != 0) {
+ /*
+ * Do LRU promotion only when we will ever shrink
+ */
+ if (e == cache->lru) {
+ cache->lru = e->prev;
+ }
+ DLIST_PROMOTE(cache->mru, e);
+ if (cache->mru == NULL) {
+ cache->mru = e;
+ }
+ }
+
+ memcache_element_parse(e, &key, value);
+ return true;
+}
+
+static void memcache_delete_element(struct memcache *cache,
+ struct memcache_element *e)
+{
+ rb_erase(&e->rb_node, &cache->tree);
+
+ if (e == cache->lru) {
+ cache->lru = e->prev;
+ }
+ DLIST_REMOVE(cache->mru, e);
+
+ cache->size -= memcache_element_size(e->keylength, e->valuelength);
+
+ SAFE_FREE(e);
+}
+
+static void memcache_trim(struct memcache *cache)
+{
+ if (cache->max_size == 0) {
+ return;
+ }
+
+ while ((cache->size > cache->max_size) && (cache->lru != NULL)) {
+ memcache_delete_element(cache, cache->lru);
+ }
+}
+
+void memcache_delete(struct memcache *cache, enum memcache_number n,
+ DATA_BLOB key)
+{
+ struct memcache_element *e;
+
+ e = memcache_find(cache, n, key);
+ if (e == NULL) {
+ return;
+ }
+
+ memcache_delete_element(cache, e);
+}
+
+void memcache_add(struct memcache *cache, enum memcache_number n,
+ DATA_BLOB key, DATA_BLOB value)
+{
+ struct memcache_element *e;
+ struct rb_node **p;
+ struct rb_node *parent;
+ DATA_BLOB cache_key, cache_value;
+ size_t element_size;
+
+ if (key.length == 0) {
+ return;
+ }
+
+ e = memcache_find(cache, n, key);
+
+ if (e != NULL) {
+ memcache_element_parse(e, &cache_key, &cache_value);
+
+ if (value.length <= cache_value.length) {
+ /*
+ * We can reuse the existing record
+ */
+ memcpy(cache_value.data, value.data, value.length);
+ e->valuelength = value.length;
+ return;
+ }
+
+ memcache_delete_element(cache, e);
+ }
+
+ element_size = memcache_element_size(key.length, value.length);
+
+
+ e = (struct memcache_element *)SMB_MALLOC(element_size);
+
+ if (e == NULL) {
+ DEBUG(0, ("malloc failed\n"));
+ return;
+ }
+
+ e->n = n;
+ e->keylength = key.length;
+ e->valuelength = value.length;
+
+ memcache_element_parse(e, &cache_key, &cache_value);
+ memcpy(cache_key.data, key.data, key.length);
+ memcpy(cache_value.data, value.data, value.length);
+
+ parent = NULL;
+ p = &cache->tree.rb_node;
+
+ while (*p) {
+ struct memcache_element *elem = memcache_node2elem(*p);
+ int cmp;
+
+ parent = (*p);
+
+ cmp = memcache_compare(elem, n, key);
+
+ p = (cmp < 0) ? &(*p)->rb_left : &(*p)->rb_right;
+ }
+
+ rb_link_node(&e->rb_node, parent, p);
+ rb_insert_color(&e->rb_node, &cache->tree);
+
+ DLIST_ADD(cache->mru, e);
+ if (cache->lru == NULL) {
+ cache->lru = e;
+ }
+
+ cache->size += element_size;
+ memcache_trim(cache);
+}
+
+void memcache_flush(struct memcache *cache, enum memcache_number n)
+{
+ struct rb_node *node;
+
+ /*
+ * Find the smallest element of number n
+ */
+
+ node = cache->tree.rb_node;
+ if (node == NULL) {
+ return;
+ }
+
+ while (true) {
+ struct memcache_element *elem = memcache_node2elem(node);
+ struct rb_node *next;
+
+ if ((int)elem->n < (int)n) {
+ next = node->rb_right;
+ }
+ else {
+ next = node->rb_left;
+ }
+ if (next == NULL) {
+ break;
+ }
+ node = next;
+ }
+
+ node = rb_next(node);
+ if (node == NULL) {
+ return;
+ }
+
+ while (node != NULL) {
+ struct memcache_element *e = memcache_node2elem(node);
+ struct rb_node *next = rb_next(node);
+
+ memcache_delete_element(cache, e);
+ node = next;
+ }
+}
diff --git a/source/param/loadparm.c b/source/param/loadparm.c
index 881bcec..eea3ece 100644
--- a/source/param/loadparm.c
+++ b/source/param/loadparm.c
@@ -88,8 +88,6 @@ static bool include_registry_globals = False;
#define USERSHARE_VALID 1
#define USERSHARE_PENDING_DELETE 2
-bool use_getwd_cache = True;
-
extern int extra_time_offset;
static bool defaults_saved = False;
@@ -215,6 +213,7 @@ typedef struct {
int pwordlevel;
int unamelevel;
int deadtime;
+ bool getwd_cache;
int maxprotocol;
int minprotocol;
int security;
@@ -1037,7 +1036,7 @@ static struct parm_struct parm_table[] = {
{"block size", P_INTEGER, P_LOCAL, &sDefault.iBlock_size, NULL, NULL, FLAG_ADVANCED | FLAG_SHARE | FLAG_GLOBAL},
{"deadtime", P_INTEGER, P_GLOBAL, &Globals.deadtime, NULL, NULL, FLAG_ADVANCED},
- {"getwd cache", P_BOOL, P_GLOBAL, &use_getwd_cache, NULL, NULL, FLAG_ADVANCED},
+ {"getwd cache", P_BOOL, P_GLOBAL, &Globals.getwd_cache, NULL, NULL, FLAG_ADVANCED},
{"keepalive", P_INTEGER, P_GLOBAL, &Globals.iKeepalive, NULL, NULL, FLAG_ADVANCED},
{"change notify", P_BOOL, P_LOCAL, &sDefault.bChangeNotify, NULL, NULL, FLAG_ADVANCED | FLAG_SHARE },
{"directory name cache size", P_INTEGER, P_LOCAL, &sDefault.iDirectoryNameCacheSize, NULL, NULL, FLAG_ADVANCED | FLAG_SHARE },
@@ -1535,6 +1534,7 @@ static void init_globals(bool first_time_only)
Globals.pwordlevel = 0;
Globals.unamelevel = 0;
Globals.deadtime = 0;
+ Globals.getwd_cache = true;
Globals.bLargeReadwrite = True;
Globals.max_log_size = 5000;
Globals.max_open_files = MAX_OPEN_FILES;
@@ -2023,6 +2023,7 @@ FN_GLOBAL_INTEGER(lp_maxmux, &Globals.max_mux)
FN_GLOBAL_INTEGER(lp_passwordlevel, &Globals.pwordlevel)
FN_GLOBAL_INTEGER(lp_usernamelevel, &Globals.unamelevel)
FN_GLOBAL_INTEGER(lp_deadtime, &Globals.deadtime)
+FN_GLOBAL_BOOL(lp_getwd_cache, &Globals.getwd_cache)
FN_GLOBAL_INTEGER(lp_maxprotocol, &Globals.maxprotocol)
FN_GLOBAL_INTEGER(lp_minprotocol, &Globals.minprotocol)
FN_GLOBAL_INTEGER(lp_security, &Globals.security)
diff --git a/source/smbd/mangle_hash2.c b/source/smbd/mangle_hash2.c
index 7066c2a..9643506 100644
--- a/source/smbd/mangle_hash2.c
+++ b/source/smbd/mangle_hash2.c
@@ -93,15 +93,6 @@ static unsigned char char_flags[256];
*/
static unsigned mangle_prefix;
-/* we will use a very simple direct mapped prefix cache. The big
- advantage of this cache structure is speed and low memory usage
-
- The cache is indexed by the low-order bits of the hash, and confirmed by
- hashing the resulting cache entry to match the known hash
-*/
-static char **prefix_cache;
-static unsigned int *prefix_cache_hashes;
-
/* these are the characters we use in the 8.3 hash. Must be 36 chars long */
static const char *basechars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
static unsigned char base_reverse[256];
@@ -147,57 +138,39 @@ static unsigned int mangle_hash(const char *key, unsigned int length)
return value & ~0x80000000;
}
-/*
- initialise (ie. allocate) the prefix cache
- */
-static bool cache_init(void)
-{
- if (prefix_cache) {
- return True;
- }
-
- prefix_cache = SMB_CALLOC_ARRAY(char *,MANGLE_CACHE_SIZE);
- if (!prefix_cache) {
- return False;
- }
-
- prefix_cache_hashes = SMB_CALLOC_ARRAY(unsigned int, MANGLE_CACHE_SIZE);
- if (!prefix_cache_hashes) {
- SAFE_FREE(prefix_cache);
- return False;
- }
-
- return True;
-}
-
/*
insert an entry into the prefix cache. The string might not be null
terminated */
static void cache_insert(const char *prefix, int length, unsigned int hash)
{
- int i = hash % MANGLE_CACHE_SIZE;
+ char *str = SMB_STRNDUP(prefix, length);
- if (prefix_cache[i]) {
- free(prefix_cache[i]);
+ if (str == NULL) {
+ return;
}
- prefix_cache[i] = SMB_STRNDUP(prefix, length);
- prefix_cache_hashes[i] = hash;
+ memcache_add(smbd_memcache(), MANGLE_HASH2_CACHE,
+ data_blob_const(&hash, sizeof(hash)),
+ data_blob_const(str, length+1));
+ SAFE_FREE(str);
}
/*
lookup an entry in the prefix cache. Return NULL if not found.
*/
-static const char *cache_lookup(unsigned int hash)
+static char *cache_lookup(TALLOC_CTX *mem_ctx, unsigned int hash)
{
- int i = hash % MANGLE_CACHE_SIZE;
+ DATA_BLOB value;
- if (!prefix_cache[i] || hash != prefix_cache_hashes[i]) {
+ if (!memcache_lookup(smbd_memcache(), MANGLE_HASH2_CACHE,
+ data_blob_const(&hash, sizeof(hash)), &value)) {
return NULL;
}
- /* yep, it matched */
- return prefix_cache[i];
+ SMB_ASSERT((value.length > 0)
+ && (value.data[value.length-1] == '\0'));
+
+ return talloc_strdup(mem_ctx, (char *)value.data);
}
@@ -377,7 +350,7 @@ static bool lookup_name_from_8_3(TALLOC_CTX *ctx,
{
unsigned int hash, multiplier;
unsigned int i;
- const char *prefix;
+ char *prefix;
char extension[4];
*pp_out = NULL;
@@ -397,7 +370,7 @@ static bool lookup_name_from_8_3(TALLOC_CTX *ctx,
}
/* now look in the prefix cache for that hash */
- prefix = cache_lookup(hash);
+ prefix = cache_lookup(ctx, hash);
if (!prefix) {
M_DEBUG(10,("lookup_name_from_8_3: %s -> %08X -> not found\n",
name, hash));
@@ -421,6 +394,8 @@ static bool lookup_name_from_8_3(TALLOC_CTX *ctx,
*pp_out = talloc_strdup(ctx, prefix);
}
+ TALLOC_FREE(prefix);
+
if (!pp_out) {
M_DEBUG(0,("talloc_fail"));
return False;
@@ -728,10 +703,6 @@ struct mangle_fns *mangle_hash2_init(void)
init_tables();
mangle_reset();
- if (!cache_init()) {
--
Samba Shared Repository
More information about the samba-cvs
mailing list