>From a5590b4763c8e8806847ac10f411eecfa4f31d4a Mon Sep 17 00:00:00 2001 From: Simo Sorce Date: Sat, 22 Sep 2012 16:15:47 -0400 Subject: [PATCH 1/2] Add memory limiting capability to talloc By calling talloc_set_memlimit() we can now set a max memory limit for a whole talloc hierarchy. ANy attempt to allocate memory beyond the max allowed for the whole hierarchy wil cause an allocation failure. Stealing memory correctly accounts for used memory in the old and the new hierarchy but exceeding the memory limit in the new parent will not cause a failure. --- lib/talloc/talloc.c | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++-- lib/talloc/talloc.h | 8 +++ lib/talloc/wscript | 2 +- 3 files changed, 159 insertions(+), 5 deletions(-) mode change 100644 => 100755 lib/talloc/wscript diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 18ee548095792a838687b421e5ad96bcb433e7ef..8c911335780fe09fa8f6d3189c25e43a18149a06 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -71,6 +71,7 @@ #define TALLOC_FLAG_LOOP 0x02 #define TALLOC_FLAG_POOL 0x04 /* This is a talloc pool */ #define TALLOC_FLAG_POOLMEM 0x08 /* This is allocated in a pool */ + #define TALLOC_MAGIC_REFERENCE ((const char *)1) /* by default we abort when given a bad pointer (such as when talloc_free() is called @@ -227,6 +228,13 @@ struct talloc_reference_handle { const char *location; }; +struct talloc_memlimit { + struct talloc_chunk *parent; + struct talloc_memlimit *upper; + size_t max_size; + size_t cur_size; +}; + typedef int (*talloc_destructor_t)(void *); struct talloc_chunk { @@ -239,6 +247,15 @@ struct talloc_chunk { unsigned flags; /* + * limit semantics: + * if 'limit' is set it means all *new* children of the context will + * be limited to a total aggregate size ox max_size for memory + * allocations. + * cur_size is used to kep track of the current use + */ + struct talloc_memlimit *limit; + + /* * "pool" has dual use: * * For the talloc pool itself (i.e. TALLOC_FLAG_POOL is set), "pool" @@ -543,6 +560,7 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, static inline void *__talloc(const void *context, size_t size) { struct talloc_chunk *tc = NULL; + struct talloc_memlimit *limit = NULL; if (unlikely(context == NULL)) { context = null_context; @@ -553,8 +571,22 @@ static inline void *__talloc(const void *context, size_t size) } if (context != NULL) { - tc = talloc_alloc_pool(talloc_chunk_from_ptr(context), - TC_HDR_SIZE+size); + struct talloc_chunk *ptc = talloc_chunk_from_ptr(context); + struct talloc_memlimit *l; + + if (ptc->limit != NULL) { + limit = ptc->limit; + } + for (l = limit; l != NULL; l = l->upper) { + if (l->max_size != 0 && + ((l->max_size <= l->cur_size) || + (l->max_size - l->cur_size < TC_HDR_SIZE+size))) { + /* exceeds max mem policy */ + return NULL; + } + } + + tc = talloc_alloc_pool(ptc, TC_HDR_SIZE+size); } if (tc == NULL) { @@ -564,6 +596,15 @@ static inline void *__talloc(const void *context, size_t size) tc->pool = NULL; } + if (limit != NULL) { + struct talloc_memlimit *l; + + for (l = limit; l != NULL; l = l->upper) { + l->cur_size += TC_HDR_SIZE+size; + } + } + + tc->limit = limit; tc->size = size; tc->destructor = NULL; tc->child = NULL; @@ -852,6 +893,24 @@ static inline int _talloc_free_internal(void *ptr, const char *location) tc->flags |= TALLOC_FLAG_FREE; + /* If we are part of a memory limited context hierarchy + * we need to subtract the memory used from the counters */ + if (tc->limit) { + struct talloc_memlimit *l; + + for (l = tc->limit; l != NULL; l = l->upper) { + if (l->cur_size >= tc->size+TC_HDR_SIZE) { + l->cur_size -= tc->size+TC_HDR_SIZE; + } /* else corrupt chain, not much we can do */ + } + + if (tc->limit->parent == tc) { + free(tc->limit); + } + + tc->limit = NULL; + } + /* we mark the freed memory with where we called the free * from. This means on a double free error we can report where * the first free came from @@ -880,6 +939,8 @@ static inline int _talloc_free_internal(void *ptr, const char *location) return 0; } +static size_t _talloc_ctx_total_size(struct talloc_chunk *parent); + /* move a lump of memory from one talloc context to another return the ptr on success, or NULL if it could not be transferred. @@ -888,6 +949,7 @@ static inline int _talloc_free_internal(void *ptr, const char *location) static void *_talloc_steal_internal(const void *new_ctx, const void *ptr) { struct talloc_chunk *tc, *new_tc; + size_t ctx_size = 0; if (unlikely(!ptr)) { return NULL; @@ -899,6 +961,23 @@ static void *_talloc_steal_internal(const void *new_ctx, const void *ptr) tc = talloc_chunk_from_ptr(ptr); + if (tc->limit != NULL) { + struct talloc_memlimit *l; + + ctx_size = _talloc_ctx_total_size(tc); + + for (l = tc->limit->upper; l != NULL; l = l->upper) { + if (l->cur_size > ctx_size) { + l->cur_size -= ctx_size; + } + } + if (tc->limit->parent == tc) { + tc->limit->upper = NULL; + } else { + tc->limit = NULL; + } + } + if (unlikely(new_ctx == NULL)) { if (tc->parent) { _TLIST_REMOVE(tc->parent->child, tc); @@ -909,7 +988,7 @@ static void *_talloc_steal_internal(const void *new_ctx, const void *ptr) if (tc->prev) tc->prev->next = tc->next; if (tc->next) tc->next->prev = tc->prev; } - + tc->parent = tc->next = tc->prev = NULL; return discard_const_p(void, ptr); } @@ -935,6 +1014,24 @@ static void *_talloc_steal_internal(const void *new_ctx, const void *ptr) if (new_tc->child) new_tc->child->parent = NULL; _TLIST_ADD(new_tc->child, tc); + if (new_tc->limit) { + struct talloc_memlimit *l; + + if (ctx_size == 0) { + ctx_size = _talloc_ctx_total_size(tc); + } + + for (l = new_tc->limit; l != NULL; l = l->upper) { + l->cur_size += ctx_size; + } + + if (tc->limit) { + tc->limit->upper = new_tc->limit; + } else { + tc->limit = new_tc->limit; + } + } + return discard_const_p(void, ptr); } @@ -1068,7 +1165,7 @@ _PUBLIC_ int talloc_unlink(const void *context, void *ptr) if (tc_c != talloc_parent_chunk(ptr)) { return -1; } - + tc_p = talloc_chunk_from_ptr(ptr); if (tc_p->refs == NULL) { @@ -2359,3 +2456,52 @@ _PUBLIC_ int talloc_is_parent(const void *context, const void *ptr) { return _talloc_is_parent(context, ptr, TALLOC_MAX_DEPTH); } + +/* + return the total size of memory used by this context and all children +*/ +static size_t _talloc_ctx_total_size(struct talloc_chunk *parent) +{ + struct talloc_chunk *tc; + size_t total; + + if (parent->limit && parent->limit->parent == parent) { + return parent->limit->cur_size; + } + + total = parent->size + TC_HDR_SIZE; + + for (tc = parent->child; tc != NULL; tc = tc->next) { + total += _talloc_ctx_total_size(tc); + } + + return total; +} + +_PUBLIC_ int talloc_set_memlimit(const void *ctx, size_t max_size) +{ + struct talloc_chunk *tc = talloc_chunk_from_ptr(ctx); + struct talloc_memlimit *limit = NULL; + + if (tc->limit && tc->limit->parent == tc) { + tc->limit->max_size = max_size; + return 0; + } + + limit = malloc(sizeof(struct talloc_memlimit)); + if (limit == NULL) { + return 1; + } + limit->parent = tc; + limit->max_size = max_size; + limit->cur_size = _talloc_ctx_total_size(tc); + + if (tc->limit) { + limit->upper = tc->limit; + } else { + limit->upper = NULL; + } + tc->limit = limit; + + return 0; +} diff --git a/lib/talloc/talloc.h b/lib/talloc/talloc.h index e48dc09a400837ff819bc9f208f209fa73466aac..83f5799be106d9db9cb7462f871db965aa13a4f2 100644 --- a/lib/talloc/talloc.h +++ b/lib/talloc/talloc.h @@ -1842,6 +1842,14 @@ void talloc_set_log_fn(void (*log_fn)(const char *message)); */ void talloc_set_log_stderr(void); +/** + * @brief Set a max memory limit for the current context hierarchy + * + * @param[in] ctx The talloc context to set the limit on + * @param[in] max_size The (new) max_size + */ +int talloc_set_memlimit(const void *ctx, size_t max_size); + /* @} ******************************************************************/ #if TALLOC_DEPRECATED diff --git a/lib/talloc/wscript b/lib/talloc/wscript old mode 100644 new mode 100755 index 447406bf6e7a4be320f32c9a2f5d5b4e2db01fe5..8d3246b26bb937cffba24c944719fd13777e4e5b --- a/lib/talloc/wscript +++ b/lib/talloc/wscript @@ -1,7 +1,7 @@ #!/usr/bin/env python APPNAME = 'talloc' -VERSION = '2.0.7' +VERSION = '2.0.8' blddir = 'bin' -- 1.7.11.4