From e123aedc25ec739a81fff2a2dc84a33251d5cb68 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Sun, 5 May 2013 17:39:45 +0200 Subject: [PATCH 01/12] talloc: Slightly simplify __talloc We don't do the for loop anyway if limit==NULL Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/talloc/talloc.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 76f0aee..9a27b78 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -565,6 +565,7 @@ static inline void *__talloc(const void *context, size_t size) { struct talloc_chunk *tc = NULL; struct talloc_memlimit *limit = NULL; + struct talloc_memlimit *l; if (unlikely(context == NULL)) { context = null_context; @@ -596,12 +597,8 @@ static inline void *__talloc(const void *context, size_t size) tc->pool = NULL; } - if (limit != NULL) { - struct talloc_memlimit *l; - - for (l = limit; l != NULL; l = l->upper) { - l->cur_size += TC_HDR_SIZE+size; - } + for (l = limit; l != NULL; l = l->upper) { + l->cur_size += TC_HDR_SIZE+size; } tc->limit = limit; -- 1.7.9.5 From 9a83f9a48def5dde43891daee5ae8245a0c61fea Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 18 Apr 2013 10:58:40 +0200 Subject: [PATCH 02/12] talloc: Decouple the dual use of chunk->pool If we want nested pools, we will have pools that are pool members. So we will have to have a separate "next object" pointer for pools. As we have struct talloc_pool_chunk now, this additional pointer does not affect normal talloc objects. Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/talloc/talloc.c | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 9a27b78..cf31d67 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -241,6 +241,8 @@ static bool talloc_memlimit_update(struct talloc_memlimit *limit, typedef int (*talloc_destructor_t)(void *); +union talloc_pool_chunk; + struct talloc_chunk { struct talloc_chunk *next, *prev; struct talloc_chunk *parent, *child; @@ -260,17 +262,12 @@ struct talloc_chunk { struct talloc_memlimit *limit; /* - * "pool" has dual use: - * - * For the talloc pool itself (i.e. TALLOC_FLAG_POOL is set), "pool" - * marks the end of the currently allocated area. - * - * For members of the pool (i.e. TALLOC_FLAG_POOLMEM is set), "pool" + * For members of a pool (i.e. TALLOC_FLAG_POOLMEM is set), "pool" * is a pointer to the struct talloc_chunk of the pool that it was * allocated from. This way children can quickly find the pool to chew * from. */ - void *pool; + union talloc_pool_chunk *pool; }; /* 16 byte alignment seems to keep everyone happy */ @@ -463,6 +460,7 @@ union talloc_pool_chunk { * on 32-bit platforms. */ struct tc_pool_hdr { struct talloc_chunk c; + void *next; unsigned int object_count; } hdr; /* This makes it always 16 byte aligned. */ @@ -476,7 +474,7 @@ static void *tc_pool_end(union talloc_pool_chunk *pool_tc) static size_t tc_pool_space_left(union talloc_pool_chunk *pool_tc) { - return (char *)tc_pool_end(pool_tc) - (char *)pool_tc->hdr.c.pool; + return (char *)tc_pool_end(pool_tc) - (char *)pool_tc->hdr.next; } static void *tc_pool_first_chunk(union talloc_pool_chunk *pool_tc) @@ -496,11 +494,11 @@ static void tc_invalidate_pool(union talloc_pool_chunk *pool_tc) size_t flen = tc_pool_space_left(pool_tc); if (unlikely(talloc_fill.enabled)) { - memset(pool_tc->hdr.c.pool, talloc_fill.fill_value, flen); + memset(pool_tc->hdr.next, talloc_fill.fill_value, flen); } #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_NOACCESS) - VALGRIND_MAKE_MEM_NOACCESS(pool_tc->hdr.c.pool, flen); + VALGRIND_MAKE_MEM_NOACCESS(pool_tc->hdr.next, flen); #endif } @@ -524,7 +522,7 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, pool_ctx = (union talloc_pool_chunk *)parent; } else if (parent->flags & TALLOC_FLAG_POOLMEM) { - pool_ctx = (union talloc_pool_chunk *)parent->pool; + pool_ctx = parent->pool; } if (pool_ctx == NULL) { @@ -542,13 +540,13 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, return NULL; } - result = (struct talloc_chunk *)pool_ctx->hdr.c.pool; + result = (struct talloc_chunk *)pool_ctx->hdr.next; #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) VALGRIND_MAKE_MEM_UNDEFINED(result, size); #endif - pool_ctx->hdr.c.pool = (void *)((char *)result + chunk_size); + pool_ctx->hdr.next = (void *)((char *)result + chunk_size); result->flags = TALLOC_MAGIC | TALLOC_FLAG_POOLMEM; result->pool = pool_ctx; @@ -650,7 +648,7 @@ _PUBLIC_ void *talloc_pool(const void *context, size_t size) return NULL; } pool_tc->hdr.c.flags |= TALLOC_FLAG_POOL; - pool_tc->hdr.c.pool = tc_pool_first_chunk(pool_tc); + pool_tc->hdr.next = tc_pool_first_chunk(pool_tc); pool_tc->hdr.object_count = 1; @@ -760,7 +758,7 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, union talloc_pool_chunk *pool; void *next_tc; - pool = (union talloc_pool_chunk *)tc->pool; + pool = tc->pool; next_tc = tc_next_chunk(tc); tc->flags |= TALLOC_FLAG_FREE; @@ -789,7 +787,7 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, * the rest is available for new objects * again. */ - pool->hdr.c.pool = tc_pool_first_chunk(pool); + pool->hdr.next = tc_pool_first_chunk(pool); tc_invalidate_pool(pool); return; } @@ -807,13 +805,13 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, return; } - if (pool->hdr.c.pool == next_tc) { + if (pool->hdr.next == next_tc) { /* * if pool->pool still points to end of * 'tc' (which is stored in the 'next_tc' variable), * we can reclaim the memory of 'tc'. */ - pool->hdr.c.pool = tc; + pool->hdr.next = tc; return; } @@ -1535,7 +1533,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons /* handle realloc inside a talloc_pool */ if (unlikely(tc->flags & TALLOC_FLAG_POOLMEM)) { - pool_tc = (union talloc_pool_chunk *)tc->pool; + pool_tc = tc->pool; } #if (ALWAYS_REALLOC == 0) @@ -1545,9 +1543,9 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons void *next_tc = tc_next_chunk(tc); TC_INVALIDATE_SHRINK_CHUNK(tc, size); tc->size = size; - if (next_tc == pool_tc->hdr.c.pool) { + if (next_tc == pool_tc->hdr.next) { /* note: tc->size has changed, so this works */ - pool_tc->hdr.c.pool = tc_next_chunk(tc); + pool_tc->hdr.next = tc_next_chunk(tc); } return ptr; } else if ((tc->size - size) < 1024) { @@ -1634,11 +1632,11 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons * because we want to invalidate the padding * too. */ - pool_tc->hdr.c.pool = new_used + (char *)new_ptr; + pool_tc->hdr.next = new_used + (char *)new_ptr; tc_invalidate_pool(pool_tc); /* now the aligned pointer */ - pool_tc->hdr.c.pool = new_chunk_size + (char *)new_ptr; + pool_tc->hdr.next = new_chunk_size + (char *)new_ptr; goto got_new_ptr; } @@ -1660,7 +1658,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons return ptr; } - if (next_tc == pool_tc->hdr.c.pool) { + if (next_tc == pool_tc->hdr.next) { /* * optimize for the case where 'tc' is the last * chunk in the pool. @@ -1679,7 +1677,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons return NULL; } tc->size = size; - pool_tc->hdr.c.pool = tc_next_chunk(tc); + pool_tc->hdr.next = tc_next_chunk(tc); return ptr; } } -- 1.7.9.5 From 36cd67516ff9854293f0bf31b158416cd40c9d95 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Sun, 5 May 2013 17:30:34 +0200 Subject: [PATCH 03/12] TODO-FIX: talloc: Introduce __talloc_with_prefix This will allow to exchange the extra talloc pool header with the talloc_chunk structure Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: l->cur_size += total_len; doesn't match the talloc_memlimit_check() arguments, as we don't record the prefix_size somewhere, we can't decrement it correctly on free, so we should ignore the size? OR: maybe even better include prefix_len in all places, should we record the prefix_len? TODO: don't we need integer wrap protection? --- lib/talloc/talloc.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index cf31d67..376da09 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -507,7 +507,7 @@ static void tc_invalidate_pool(union talloc_pool_chunk *pool_tc) */ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, - size_t size) + size_t size, size_t prefix_len) { union talloc_pool_chunk *pool_ctx = NULL; size_t space_left; @@ -534,13 +534,14 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, /* * Align size to 16 bytes */ - chunk_size = TC_ALIGN16(size); + chunk_size = TC_ALIGN16(size + prefix_len); if (space_left < chunk_size) { return NULL; } - result = (struct talloc_chunk *)pool_ctx->hdr.next; + result = (struct talloc_chunk *) + ((char *)pool_ctx->hdr.next + prefix_len); #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) VALGRIND_MAKE_MEM_UNDEFINED(result, size); @@ -559,11 +560,13 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, /* Allocate a bit of memory as a child of an existing pointer */ -static inline void *__talloc(const void *context, size_t size) +static inline void *__talloc_with_prefix(const void *context, size_t size, + size_t prefix_len) { struct talloc_chunk *tc = NULL; struct talloc_memlimit *limit = NULL; struct talloc_memlimit *l; + size_t total_len; if (unlikely(context == NULL)) { context = null_context; @@ -573,6 +576,8 @@ static inline void *__talloc(const void *context, size_t size) return NULL; } + total_len = TC_HDR_SIZE + size; + if (context != NULL) { struct talloc_chunk *ptc = talloc_chunk_from_ptr(context); @@ -580,23 +585,25 @@ static inline void *__talloc(const void *context, size_t size) limit = ptc->limit; } - if (!talloc_memlimit_check(limit, (TC_HDR_SIZE+size))) { + if (!talloc_memlimit_check(limit, total_len)) { errno = ENOMEM; return NULL; } - tc = talloc_alloc_pool(ptc, TC_HDR_SIZE+size); + tc = talloc_alloc_pool(ptc, total_len, prefix_len); } + total_len += prefix_len; + if (tc == NULL) { - tc = (struct talloc_chunk *)malloc(TC_HDR_SIZE+size); + tc = (struct talloc_chunk *)malloc(total_len); if (unlikely(tc == NULL)) return NULL; tc->flags = TALLOC_MAGIC; tc->pool = NULL; } for (l = limit; l != NULL; l = l->upper) { - l->cur_size += TC_HDR_SIZE+size; + l->cur_size += total_len; } tc->limit = limit; @@ -626,6 +633,11 @@ static inline void *__talloc(const void *context, size_t size) return TC_PTR_FROM_CHUNK(tc); } +static inline void *__talloc(const void *context, size_t size) +{ + return __talloc_with_prefix(context, size, 0); +} + /* * Create a talloc pool */ @@ -1577,7 +1589,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons #if ALWAYS_REALLOC if (pool_tc) { - new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE); + new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE, 0); pool_tc->hdr.object_count--; if (new_ptr == NULL) { @@ -1682,7 +1694,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons } } - new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE); + new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE, 0); if (new_ptr == NULL) { new_ptr = malloc(TC_HDR_SIZE+size); -- 1.7.9.5 From 3d13b4f16d67afde03b32a3338cc762ff9981d17 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 8 May 2013 14:26:48 +0200 Subject: [PATCH 04/12] TODO-SPLIT: talloc: Put pool-specific data before the chunk This is a preparation to make talloc pool real objects themselves Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: isn't the VALGRIND_MAKE_MEM_UNDEFINED(new_ptr, undef_len); change generic and correct before the other changes? If so it should be a commit on its own. Otherwise Reviewed-by: Stefan Metzmacher on both commits... --- lib/talloc/talloc.c | 197 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 120 insertions(+), 77 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 376da09..38bfc22 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -241,7 +241,7 @@ static bool talloc_memlimit_update(struct talloc_memlimit *limit, typedef int (*talloc_destructor_t)(void *); -union talloc_pool_chunk; +struct talloc_pool_hdr; struct talloc_chunk { struct talloc_chunk *next, *prev; @@ -267,7 +267,7 @@ struct talloc_chunk { * allocated from. This way children can quickly find the pool to chew * from. */ - union talloc_pool_chunk *pool; + struct talloc_pool_hdr *pool; }; /* 16 byte alignment seems to keep everyone happy */ @@ -455,31 +455,37 @@ _PUBLIC_ const char *talloc_parent_name(const void *ptr) memory footprint of each talloc chunk by those 16 bytes. */ -union talloc_pool_chunk { - /* This lets object_count nestle into 16-byte padding of talloc_chunk, - * on 32-bit platforms. */ - struct tc_pool_hdr { - struct talloc_chunk c; - void *next; - unsigned int object_count; - } hdr; - /* This makes it always 16 byte aligned. */ - char pad[TC_ALIGN16(sizeof(struct tc_pool_hdr))]; +struct talloc_pool_hdr { + void *end; + unsigned int object_count; }; -static void *tc_pool_end(union talloc_pool_chunk *pool_tc) +#define TP_HDR_SIZE TC_ALIGN16(sizeof(struct talloc_pool_hdr)) + +static struct talloc_pool_hdr *talloc_pool_from_chunk(struct talloc_chunk *c) +{ + return (struct talloc_pool_hdr *)((char *)c - TP_HDR_SIZE); +} + +static struct talloc_chunk *talloc_chunk_from_pool(struct talloc_pool_hdr *h) { - return (char *)pool_tc + TC_HDR_SIZE + pool_tc->hdr.c.size; + return (struct talloc_chunk *)((char *)h + TP_HDR_SIZE); } -static size_t tc_pool_space_left(union talloc_pool_chunk *pool_tc) +static void *tc_pool_end(struct talloc_pool_hdr *pool_hdr) { - return (char *)tc_pool_end(pool_tc) - (char *)pool_tc->hdr.next; + struct talloc_chunk *tc = talloc_chunk_from_pool(pool_hdr); + return (char *)tc + TC_HDR_SIZE + tc->size; } -static void *tc_pool_first_chunk(union talloc_pool_chunk *pool_tc) +static size_t tc_pool_space_left(struct talloc_pool_hdr *pool_hdr) { - return pool_tc + 1; + return (char *)tc_pool_end(pool_hdr) - (char *)pool_hdr->end; +} + +static void *tc_pool_first_chunk(struct talloc_pool_hdr *pool_hdr) +{ + return TC_PTR_FROM_CHUNK(talloc_chunk_from_pool(pool_hdr)); } /* If tc is inside a pool, this gives the next neighbour. */ @@ -489,16 +495,16 @@ static void *tc_next_chunk(struct talloc_chunk *tc) } /* Mark the whole remaining pool as not accessable */ -static void tc_invalidate_pool(union talloc_pool_chunk *pool_tc) +static void tc_invalidate_pool(struct talloc_pool_hdr *pool_hdr) { - size_t flen = tc_pool_space_left(pool_tc); + size_t flen = tc_pool_space_left(pool_hdr); if (unlikely(talloc_fill.enabled)) { - memset(pool_tc->hdr.next, talloc_fill.fill_value, flen); + memset(pool_hdr->end, talloc_fill.fill_value, flen); } #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_NOACCESS) - VALGRIND_MAKE_MEM_NOACCESS(pool_tc->hdr.next, flen); + VALGRIND_MAKE_MEM_NOACCESS(pool_hdr->end, flen); #endif } @@ -509,7 +515,7 @@ static void tc_invalidate_pool(union talloc_pool_chunk *pool_tc) static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, size_t size, size_t prefix_len) { - union talloc_pool_chunk *pool_ctx = NULL; + struct talloc_pool_hdr *pool_hdr = NULL; size_t space_left; struct talloc_chunk *result; size_t chunk_size; @@ -519,17 +525,17 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, } if (parent->flags & TALLOC_FLAG_POOL) { - pool_ctx = (union talloc_pool_chunk *)parent; + pool_hdr = talloc_pool_from_chunk(parent); } else if (parent->flags & TALLOC_FLAG_POOLMEM) { - pool_ctx = parent->pool; + pool_hdr = parent->pool; } - if (pool_ctx == NULL) { + if (pool_hdr == NULL) { return NULL; } - space_left = tc_pool_space_left(pool_ctx); + space_left = tc_pool_space_left(pool_hdr); /* * Align size to 16 bytes @@ -540,19 +546,18 @@ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, return NULL; } - result = (struct talloc_chunk *) - ((char *)pool_ctx->hdr.next + prefix_len); + result = (struct talloc_chunk *)((char *)pool_hdr->end + prefix_len); #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) - VALGRIND_MAKE_MEM_UNDEFINED(result, size); + VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, chunk_size); #endif - pool_ctx->hdr.next = (void *)((char *)result + chunk_size); + pool_hdr->end = (void *)((char *)pool_hdr->end + chunk_size); result->flags = TALLOC_MAGIC | TALLOC_FLAG_POOLMEM; - result->pool = pool_ctx; + result->pool = pool_hdr; - pool_ctx->hdr.object_count++; + pool_hdr->object_count++; return result; } @@ -596,8 +601,13 @@ static inline void *__talloc_with_prefix(const void *context, size_t size, total_len += prefix_len; if (tc == NULL) { - tc = (struct talloc_chunk *)malloc(total_len); - if (unlikely(tc == NULL)) return NULL; + char *ptr = malloc(total_len); + + if (unlikely(ptr == NULL)) { + return NULL; + } + + tc = (struct talloc_chunk *)(ptr + prefix_len); tc->flags = TALLOC_MAGIC; tc->pool = NULL; } @@ -644,27 +654,32 @@ static inline void *__talloc(const void *context, size_t size) _PUBLIC_ void *talloc_pool(const void *context, size_t size) { - union talloc_pool_chunk *pool_tc; - void *result = __talloc(context, sizeof(*pool_tc) - TC_HDR_SIZE + size); + struct talloc_chunk *tc; + struct talloc_pool_hdr *pool_hdr; + void *result; + + result = __talloc_with_prefix(context, size, TP_HDR_SIZE); if (unlikely(result == NULL)) { return NULL; } - pool_tc = (union talloc_pool_chunk *)talloc_chunk_from_ptr(result); - if (unlikely(pool_tc->hdr.c.flags & TALLOC_FLAG_POOLMEM)) { + tc = talloc_chunk_from_ptr(result); + pool_hdr = talloc_pool_from_chunk(tc); + + if (unlikely(tc->flags & TALLOC_FLAG_POOLMEM)) { /* We don't handle this correctly, so fail. */ talloc_log("talloc: cannot allocate pool off another pool %s\n", talloc_get_name(context)); talloc_free(result); return NULL; } - pool_tc->hdr.c.flags |= TALLOC_FLAG_POOL; - pool_tc->hdr.next = tc_pool_first_chunk(pool_tc); + tc->flags |= TALLOC_FLAG_POOL; - pool_tc->hdr.object_count = 1; + pool_hdr->object_count = 1; + pool_hdr->end = result; - tc_invalidate_pool(pool_tc); + tc_invalidate_pool(pool_hdr); return result; } @@ -767,10 +782,12 @@ static void *_talloc_steal_internal(const void *new_ctx, const void *ptr); static inline void _talloc_free_poolmem(struct talloc_chunk *tc, const char *location) { - union talloc_pool_chunk *pool; + struct talloc_pool_hdr *pool; + struct talloc_chunk *pool_tc; void *next_tc; pool = tc->pool; + pool_tc = talloc_chunk_from_pool(pool); next_tc = tc_next_chunk(tc); tc->flags |= TALLOC_FLAG_FREE; @@ -783,15 +800,15 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, TC_INVALIDATE_FULL_CHUNK(tc); - if (unlikely(pool->hdr.object_count == 0)) { + if (unlikely(pool->object_count == 0)) { talloc_abort("Pool object count zero!"); return; } - pool->hdr.object_count--; + pool->object_count--; - if (unlikely(pool->hdr.object_count == 1 - && !(pool->hdr.c.flags & TALLOC_FLAG_FREE))) { + if (unlikely(pool->object_count == 1 + && !(pool_tc->flags & TALLOC_FLAG_FREE))) { /* * if there is just one object left in the pool * and pool->flags does not have TALLOC_FLAG_FREE, @@ -799,31 +816,31 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, * the rest is available for new objects * again. */ - pool->hdr.next = tc_pool_first_chunk(pool); + pool->end = tc_pool_first_chunk(pool); tc_invalidate_pool(pool); return; } - if (unlikely(pool->hdr.object_count == 0)) { + if (unlikely(pool->object_count == 0)) { /* * we mark the freed memory with where we called the free * from. This means on a double free error we can report where * the first free came from */ - pool->hdr.c.name = location; + pool_tc->name = location; - TC_INVALIDATE_FULL_CHUNK(&pool->hdr.c); + TC_INVALIDATE_FULL_CHUNK(pool_tc); free(pool); return; } - if (pool->hdr.next == next_tc) { + if (pool->end == next_tc) { /* * if pool->pool still points to end of * 'tc' (which is stored in the 'next_tc' variable), * we can reclaim the memory of 'tc'. */ - pool->hdr.next = tc; + pool->end = tc; return; } @@ -942,21 +959,23 @@ static inline int _talloc_free_internal(void *ptr, const char *location) tc->name = location; if (tc->flags & TALLOC_FLAG_POOL) { - union talloc_pool_chunk *pool = (union talloc_pool_chunk *)tc; + struct talloc_pool_hdr *pool; + + pool = talloc_pool_from_chunk(tc); - if (unlikely(pool->hdr.object_count == 0)) { + if (unlikely(pool->object_count == 0)) { talloc_abort("Pool object count zero!"); return 0; } - pool->hdr.object_count--; + pool->object_count--; - if (likely(pool->hdr.object_count != 0)) { + if (likely(pool->object_count != 0)) { return 0; } TC_INVALIDATE_FULL_CHUNK(tc); - free(tc); + free(pool); return 0; } @@ -1507,7 +1526,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons struct talloc_chunk *tc; void *new_ptr; bool malloced = false; - union talloc_pool_chunk *pool_tc = NULL; + struct talloc_pool_hdr *pool_hdr = NULL; /* size zero is equivalent to free() */ if (unlikely(size == 0)) { @@ -1545,19 +1564,19 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons /* handle realloc inside a talloc_pool */ if (unlikely(tc->flags & TALLOC_FLAG_POOLMEM)) { - pool_tc = tc->pool; + pool_hdr = tc->pool; } #if (ALWAYS_REALLOC == 0) /* don't shrink if we have less than 1k to gain */ if (size < tc->size && tc->limit == NULL) { - if (pool_tc) { + if (pool_hdr) { void *next_tc = tc_next_chunk(tc); TC_INVALIDATE_SHRINK_CHUNK(tc, size); tc->size = size; - if (next_tc == pool_tc->hdr.next) { + if (next_tc == pool_hdr->end) { /* note: tc->size has changed, so this works */ - pool_tc->hdr.next = tc_next_chunk(tc); + pool_hdr->end = tc_next_chunk(tc); } return ptr; } else if ((tc->size - size) < 1024) { @@ -1588,9 +1607,9 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons tc->flags |= TALLOC_FLAG_FREE; #if ALWAYS_REALLOC - if (pool_tc) { + if (pool_hdr) { new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE, 0); - pool_tc->hdr.object_count--; + pool_hdr->object_count--; if (new_ptr == NULL) { new_ptr = malloc(TC_HDR_SIZE+size); @@ -1609,15 +1628,17 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons } } #else - if (pool_tc) { + if (pool_hdr) { + struct talloc_chunk *pool_tc; void *next_tc = tc_next_chunk(tc); size_t old_chunk_size = TC_ALIGN16(TC_HDR_SIZE + tc->size); size_t new_chunk_size = TC_ALIGN16(TC_HDR_SIZE + size); size_t space_needed; size_t space_left; - unsigned int chunk_count = pool_tc->hdr.object_count; + unsigned int chunk_count = pool_hdr->object_count; - if (!(pool_tc->hdr.c.flags & TALLOC_FLAG_FREE)) { + pool_tc = talloc_chunk_from_pool(pool_hdr); + if (!(pool_tc->flags & TALLOC_FLAG_FREE)) { chunk_count -= 1; } @@ -1626,14 +1647,35 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons * optimize for the case where 'tc' is the only * chunk in the pool. */ - char *start = tc_pool_first_chunk(pool_tc); + char *start = tc_pool_first_chunk(pool_hdr); space_needed = new_chunk_size; - space_left = (char *)tc_pool_end(pool_tc) - start; + space_left = (char *)tc_pool_end(pool_hdr) - start; if (space_left >= space_needed) { size_t old_used = TC_HDR_SIZE + tc->size; size_t new_used = TC_HDR_SIZE + size; new_ptr = start; + +#if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) + { + /* + * The area from + * start -> tc may have + * been freed and thus been marked as + * VALGRIND_MEM_NOACCESS. Set it to + * VALGRIND_MEM_UNDEFINED so we can + * copy into it without valgrind errors. + * We can't just mark + * new_ptr -> new_ptr + old_used + * as this may overlap on top of tc, + * (which is why we use memmove, not + * memcpy below) hence the MIN. + */ + size_t undef_len = MIN((((char *)tc) - ((char *)new_ptr)),old_used); + VALGRIND_MAKE_MEM_UNDEFINED(new_ptr, undef_len); + } +#endif + memmove(new_ptr, tc, old_used); tc = (struct talloc_chunk *)new_ptr; @@ -1644,11 +1686,12 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons * because we want to invalidate the padding * too. */ - pool_tc->hdr.next = new_used + (char *)new_ptr; - tc_invalidate_pool(pool_tc); + pool_hdr->end = new_used + (char *)new_ptr; + tc_invalidate_pool(pool_hdr); /* now the aligned pointer */ - pool_tc->hdr.next = new_chunk_size + (char *)new_ptr; + pool_hdr->end = + new_chunk_size + (char *)new_ptr; goto got_new_ptr; } @@ -1670,13 +1713,13 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons return ptr; } - if (next_tc == pool_tc->hdr.next) { + if (next_tc == pool_hdr->end) { /* * optimize for the case where 'tc' is the last * chunk in the pool. */ space_needed = new_chunk_size - old_chunk_size; - space_left = tc_pool_space_left(pool_tc); + space_left = tc_pool_space_left(pool_hdr); if (space_left >= space_needed) { TC_UNDEFINE_GROW_CHUNK(tc, size); @@ -1689,7 +1732,7 @@ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, cons return NULL; } tc->size = size; - pool_tc->hdr.next = tc_next_chunk(tc); + pool_hdr->end = tc_next_chunk(tc); return ptr; } } -- 1.7.9.5 From 5d5aba7243c8b7d9ab7195b71daf95cd88a55b9c Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 22 May 2013 17:18:01 +0200 Subject: [PATCH 05/12] talloc: Add a separate pool size This is necessary to allow talloc pools to be objects on their own. It is an incompatible change in the sense that talloc_get_size(pool) now returns 0 instead of the pool size. When the talloc_pooled_object() call is added, this will start to make sense again. Maybe we should add a talloc_pool_size call? Or is that overkill? Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/talloc/talloc.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 38bfc22..9a59af5 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -458,6 +458,7 @@ _PUBLIC_ const char *talloc_parent_name(const void *ptr) struct talloc_pool_hdr { void *end; unsigned int object_count; + size_t poolsize; }; #define TP_HDR_SIZE TC_ALIGN16(sizeof(struct talloc_pool_hdr)) @@ -475,7 +476,7 @@ static struct talloc_chunk *talloc_chunk_from_pool(struct talloc_pool_hdr *h) static void *tc_pool_end(struct talloc_pool_hdr *pool_hdr) { struct talloc_chunk *tc = talloc_chunk_from_pool(pool_hdr); - return (char *)tc + TC_HDR_SIZE + tc->size; + return (char *)tc + TC_HDR_SIZE + pool_hdr->poolsize; } static size_t tc_pool_space_left(struct talloc_pool_hdr *pool_hdr) @@ -483,17 +484,18 @@ static size_t tc_pool_space_left(struct talloc_pool_hdr *pool_hdr) return (char *)tc_pool_end(pool_hdr) - (char *)pool_hdr->end; } -static void *tc_pool_first_chunk(struct talloc_pool_hdr *pool_hdr) -{ - return TC_PTR_FROM_CHUNK(talloc_chunk_from_pool(pool_hdr)); -} - /* If tc is inside a pool, this gives the next neighbour. */ static void *tc_next_chunk(struct talloc_chunk *tc) { return (char *)tc + TC_ALIGN16(TC_HDR_SIZE + tc->size); } +static void *tc_pool_first_chunk(struct talloc_pool_hdr *pool_hdr) +{ + struct talloc_chunk *tc = talloc_chunk_from_pool(pool_hdr); + return tc_next_chunk(tc); +} + /* Mark the whole remaining pool as not accessable */ static void tc_invalidate_pool(struct talloc_pool_hdr *pool_hdr) { @@ -674,10 +676,13 @@ _PUBLIC_ void *talloc_pool(const void *context, size_t size) talloc_free(result); return NULL; } + tc->flags |= TALLOC_FLAG_POOL; + tc->size = 0; pool_hdr->object_count = 1; pool_hdr->end = result; + pool_hdr->poolsize = size; tc_invalidate_pool(pool_hdr); -- 1.7.9.5 From 4b475b176d6efb8b8e62d61786806bbea021b57c Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 9 May 2013 13:30:52 +0200 Subject: [PATCH 06/12] TODO-SQUASH: talloc: Allow nested pools Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: this removes TC_INVALIDATE_FULL_CHUNK(pool_tc); before free(pool); this should be readded... --- lib/talloc/talloc.c | 29 +++++++++++++++-------------- lib/talloc/testsuite.c | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 9a59af5..56d07b3 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -669,14 +669,6 @@ _PUBLIC_ void *talloc_pool(const void *context, size_t size) tc = talloc_chunk_from_ptr(result); pool_hdr = talloc_pool_from_chunk(tc); - if (unlikely(tc->flags & TALLOC_FLAG_POOLMEM)) { - /* We don't handle this correctly, so fail. */ - talloc_log("talloc: cannot allocate pool off another pool %s\n", - talloc_get_name(context)); - talloc_free(result); - return NULL; - } - tc->flags |= TALLOC_FLAG_POOL; tc->size = 0; @@ -834,8 +826,11 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, */ pool_tc->name = location; - TC_INVALIDATE_FULL_CHUNK(pool_tc); - free(pool); + if (pool_tc->flags & TALLOC_FLAG_POOLMEM) { + _talloc_free_poolmem(pool_tc, location); + } else { + free(pool); + } return; } @@ -865,6 +860,7 @@ static inline void _talloc_free_children_internal(struct talloc_chunk *tc, static inline int _talloc_free_internal(void *ptr, const char *location) { struct talloc_chunk *tc; + void *ptr_to_free; if (unlikely(ptr == NULL)) { return -1; @@ -979,9 +975,14 @@ static inline int _talloc_free_internal(void *ptr, const char *location) return 0; } - TC_INVALIDATE_FULL_CHUNK(tc); - free(pool); - return 0; + /* + * With object_count==0, a pool becomes a normal piece of + * memory to free. If it's allocated inside a pool, it needs + * to be freed as poolmem, else it needs to be just freed. + */ + ptr_to_free = pool; + } else { + ptr_to_free = tc; } if (tc->flags & TALLOC_FLAG_POOLMEM) { @@ -990,7 +991,7 @@ static inline int _talloc_free_internal(void *ptr, const char *location) } TC_INVALIDATE_FULL_CHUNK(tc); - free(tc); + free(ptr_to_free); return 0; } diff --git a/lib/talloc/testsuite.c b/lib/talloc/testsuite.c index d456cbb..5501d17 100644 --- a/lib/talloc/testsuite.c +++ b/lib/talloc/testsuite.c @@ -1267,6 +1267,30 @@ static bool test_pool_steal(void) return true; } +static bool test_pool_nest(void) +{ + void *p1, *p2, *p3; + void *e = talloc_new(NULL); + + p1 = talloc_pool(NULL, 1024); + torture_assert("talloc_pool", p1 != NULL, "failed"); + + p2 = talloc_pool(p1, 500); + torture_assert("talloc_pool", p2 != NULL, "failed"); + + p3 = talloc_size(p2, 10); + + talloc_steal(e, p3); + + talloc_free(p2); + + talloc_free(p3); + + talloc_free(p1); + + return true; +} + static bool test_free_ref_null_context(void) { void *p1, *p2, *p3; @@ -1540,6 +1564,8 @@ bool torture_local_talloc(struct torture_context *tctx) setlinebuf(stdout); test_reset(); + ret &= test_pool_nest(); + test_reset(); ret &= test_ref1(); test_reset(); ret &= test_ref2(); -- 1.7.9.5 From 77f687b495f5b589cb305afdc379d17043705edc Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 26 Aug 2013 16:17:19 +0200 Subject: [PATCH 07/12] sq talloc: Allow nested pools With this squash I'm happy with the commit... Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/talloc/talloc.c | 1 + lib/talloc/talloc.h | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 56d07b3..a112b3a 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -829,6 +829,7 @@ static inline void _talloc_free_poolmem(struct talloc_chunk *tc, if (pool_tc->flags & TALLOC_FLAG_POOLMEM) { _talloc_free_poolmem(pool_tc, location); } else { + TC_INVALIDATE_FULL_CHUNK(pool_tc); free(pool); } return; diff --git a/lib/talloc/talloc.h b/lib/talloc/talloc.h index f3cbcd0..aa9864b 100644 --- a/lib/talloc/talloc.h +++ b/lib/talloc/talloc.h @@ -839,8 +839,7 @@ void *talloc_find_parent_bytype(const void *ptr, #type); * talloc pool to a talloc parent outside the pool, the whole pool memory is * not free(3)'ed until that moved chunk is also talloc_free()ed. * - * @param[in] context The talloc context to hang the result off (must not - * be another pool). + * @param[in] context The talloc context to hang the result off. * * @param[in] size Size of the talloc pool. * -- 1.7.9.5 From 7f8d187dd4d75121866757552c5314a360904b3f Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 9 May 2013 13:54:51 +0200 Subject: [PATCH 08/12] TODO-SQUASH: talloc: Add talloc_pooled_object Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: as we changed the memory layout, we should change to talloc-2.1.0 which changes TALLOC_MAGIC... TODO: talloc_pooled_object() needs documentation TODO: the names of unsigned num_members, size_t member_size are confusing, I'd expect the total_size = num_members * member_size; --- lib/talloc/ABI/pytalloc-util-2.0.9.sigs | 6 +++ lib/talloc/ABI/talloc-2.0.9.sigs | 64 +++++++++++++++++++++++++++++++ lib/talloc/talloc.c | 58 ++++++++++++++++++++++++++++ lib/talloc/talloc.h | 11 ++++++ lib/talloc/wscript | 2 +- 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 lib/talloc/ABI/pytalloc-util-2.0.9.sigs create mode 100644 lib/talloc/ABI/talloc-2.0.9.sigs diff --git a/lib/talloc/ABI/pytalloc-util-2.0.9.sigs b/lib/talloc/ABI/pytalloc-util-2.0.9.sigs new file mode 100644 index 0000000..961c1a8 --- /dev/null +++ b/lib/talloc/ABI/pytalloc-util-2.0.9.sigs @@ -0,0 +1,6 @@ +pytalloc_CObject_FromTallocPtr: PyObject *(void *) +pytalloc_Check: int (PyObject *) +pytalloc_GetObjectType: PyTypeObject *(void) +pytalloc_reference_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) +pytalloc_steal: PyObject *(PyTypeObject *, void *) +pytalloc_steal_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) diff --git a/lib/talloc/ABI/talloc-2.0.9.sigs b/lib/talloc/ABI/talloc-2.0.9.sigs new file mode 100644 index 0000000..eae12cc --- /dev/null +++ b/lib/talloc/ABI/talloc-2.0.9.sigs @@ -0,0 +1,64 @@ +_talloc: void *(const void *, size_t) +_talloc_array: void *(const void *, size_t, unsigned int, const char *) +_talloc_free: int (void *, const char *) +_talloc_get_type_abort: void *(const void *, const char *, const char *) +_talloc_memdup: void *(const void *, const void *, size_t, const char *) +_talloc_move: void *(const void *, const void *) +_talloc_pooled_object: void *(const void *, size_t, const char *, unsigned int, size_t) +_talloc_realloc: void *(const void *, void *, size_t, const char *) +_talloc_realloc_array: void *(const void *, void *, size_t, unsigned int, const char *) +_talloc_reference_loc: void *(const void *, const void *, const char *) +_talloc_set_destructor: void (const void *, int (*)(void *)) +_talloc_steal_loc: void *(const void *, const void *, const char *) +_talloc_zero: void *(const void *, size_t, const char *) +_talloc_zero_array: void *(const void *, size_t, unsigned int, const char *) +talloc_asprintf: char *(const void *, const char *, ...) +talloc_asprintf_append: char *(char *, const char *, ...) +talloc_asprintf_append_buffer: char *(char *, const char *, ...) +talloc_autofree_context: void *(void) +talloc_check_name: void *(const void *, const char *) +talloc_disable_null_tracking: void (void) +talloc_enable_leak_report: void (void) +talloc_enable_leak_report_full: void (void) +talloc_enable_null_tracking: void (void) +talloc_enable_null_tracking_no_autofree: void (void) +talloc_find_parent_byname: void *(const void *, const char *) +talloc_free_children: void (void *) +talloc_get_name: const char *(const void *) +talloc_get_size: size_t (const void *) +talloc_increase_ref_count: int (const void *) +talloc_init: void *(const char *, ...) +talloc_is_parent: int (const void *, const void *) +talloc_named: void *(const void *, size_t, const char *, ...) +talloc_named_const: void *(const void *, size_t, const char *) +talloc_parent: void *(const void *) +talloc_parent_name: const char *(const void *) +talloc_pool: void *(const void *, size_t) +talloc_realloc_fn: void *(const void *, void *, size_t) +talloc_reference_count: size_t (const void *) +talloc_reparent: void *(const void *, const void *, const void *) +talloc_report: void (const void *, FILE *) +talloc_report_depth_cb: void (const void *, int, int, void (*)(const void *, int, int, int, void *), void *) +talloc_report_depth_file: void (const void *, int, int, FILE *) +talloc_report_full: void (const void *, FILE *) +talloc_set_abort_fn: void (void (*)(const char *)) +talloc_set_log_fn: void (void (*)(const char *)) +talloc_set_log_stderr: void (void) +talloc_set_memlimit: int (const void *, size_t) +talloc_set_name: const char *(const void *, const char *, ...) +talloc_set_name_const: void (const void *, const char *) +talloc_show_parents: void (const void *, FILE *) +talloc_strdup: char *(const void *, const char *) +talloc_strdup_append: char *(char *, const char *) +talloc_strdup_append_buffer: char *(char *, const char *) +talloc_strndup: char *(const void *, const char *, size_t) +talloc_strndup_append: char *(char *, const char *, size_t) +talloc_strndup_append_buffer: char *(char *, const char *, size_t) +talloc_total_blocks: size_t (const void *) +talloc_total_size: size_t (const void *) +talloc_unlink: int (const void *, void *) +talloc_vasprintf: char *(const void *, const char *, va_list) +talloc_vasprintf_append: char *(char *, const char *, va_list) +talloc_vasprintf_append_buffer: char *(char *, const char *, va_list) +talloc_version_major: int (void) +talloc_version_minor: int (void) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index a112b3a..9229e9b 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -681,6 +681,64 @@ _PUBLIC_ void *talloc_pool(const void *context, size_t size) return result; } +_PUBLIC_ void *_talloc_pooled_object(const void *ctx, size_t size, + const char *name, + unsigned num_members, size_t member_size) +{ + size_t poolsize, member_slack, tmp; + struct talloc_chunk *tc; + struct talloc_pool_hdr *pool_hdr; + void *ret; + + poolsize = size + member_size; + + if ((poolsize < size) || (poolsize < member_size)) { + goto overflow; + } + + if (num_members == UINT_MAX) { + goto overflow; + } + num_members += 1; /* the object body itself */ + + /* + * Alignment can increase the pool size by at most 15 bytes per object + * plus alignment for the object itself + */ + member_slack = (TC_HDR_SIZE + TP_HDR_SIZE + 15) * num_members; + if (member_slack < num_members) { + goto overflow; + } + + tmp = poolsize + member_slack; + if ((tmp < poolsize) || (tmp < member_slack)) { + goto overflow; + } + poolsize = tmp; + + ret = talloc_pool(ctx, poolsize); + if (ret == NULL) { + return NULL; + } + + tc = talloc_chunk_from_ptr(ret); + tc->size = size; + + pool_hdr = talloc_pool_from_chunk(tc); + +#if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) + VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, size); +#endif + + pool_hdr->end = ((char *)pool_hdr->end + TC_ALIGN16(size)); + + talloc_set_name_const(ret, name); + return ret; + +overflow: + return NULL; +} + /* setup a destructor to be called on free of a pointer the destructor should return 0 on success, or -1 on failure. diff --git a/lib/talloc/talloc.h b/lib/talloc/talloc.h index aa9864b..5e27484 100644 --- a/lib/talloc/talloc.h +++ b/lib/talloc/talloc.h @@ -847,6 +847,17 @@ void *talloc_find_parent_bytype(const void *ptr, #type); */ void *talloc_pool(const void *context, size_t size); +#ifdef DOXYGEN +void *talloc_pooled_object(const void *ctx, #type, + unsigned num_members, size_t member_size); +#else +#define talloc_pooled_object(_ctx, _type, _num_members, _member_size) \ + (_type *)_talloc_pooled_object((_ctx), sizeof(_type), #_type, \ + (_num_members), (_member_size)) +void *_talloc_pooled_object(const void *ctx, size_t size, const char *name, + unsigned num_members, size_t member_size); +#endif + /** * @brief Free a talloc chunk and NULL out the pointer. * diff --git a/lib/talloc/wscript b/lib/talloc/wscript index ecc5e24..8c13b1d 100644 --- a/lib/talloc/wscript +++ b/lib/talloc/wscript @@ -1,7 +1,7 @@ #!/usr/bin/env python APPNAME = 'talloc' -VERSION = '2.0.8' +VERSION = '2.0.9' blddir = 'bin' -- 1.7.9.5 From 6fd75ae3541fc0f57711179e73ea9199dd4b8cec Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 26 Aug 2013 16:19:08 +0200 Subject: [PATCH 09/12] sq talloc: Add talloc_pooled_object --- lib/talloc/ABI/pytalloc-util-2.0.9.sigs | 6 --- lib/talloc/ABI/pytalloc-util-2.1.0.sigs | 6 +++ lib/talloc/ABI/talloc-2.0.9.sigs | 64 ------------------------------- lib/talloc/ABI/talloc-2.1.0.sigs | 64 +++++++++++++++++++++++++++++++ lib/talloc/talloc.c | 34 ++++++++-------- lib/talloc/talloc.h | 36 ++++++++++++++--- lib/talloc/wscript | 2 +- 7 files changed, 120 insertions(+), 92 deletions(-) delete mode 100644 lib/talloc/ABI/pytalloc-util-2.0.9.sigs create mode 100644 lib/talloc/ABI/pytalloc-util-2.1.0.sigs delete mode 100644 lib/talloc/ABI/talloc-2.0.9.sigs create mode 100644 lib/talloc/ABI/talloc-2.1.0.sigs diff --git a/lib/talloc/ABI/pytalloc-util-2.0.9.sigs b/lib/talloc/ABI/pytalloc-util-2.0.9.sigs deleted file mode 100644 index 961c1a8..0000000 --- a/lib/talloc/ABI/pytalloc-util-2.0.9.sigs +++ /dev/null @@ -1,6 +0,0 @@ -pytalloc_CObject_FromTallocPtr: PyObject *(void *) -pytalloc_Check: int (PyObject *) -pytalloc_GetObjectType: PyTypeObject *(void) -pytalloc_reference_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) -pytalloc_steal: PyObject *(PyTypeObject *, void *) -pytalloc_steal_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) diff --git a/lib/talloc/ABI/pytalloc-util-2.1.0.sigs b/lib/talloc/ABI/pytalloc-util-2.1.0.sigs new file mode 100644 index 0000000..961c1a8 --- /dev/null +++ b/lib/talloc/ABI/pytalloc-util-2.1.0.sigs @@ -0,0 +1,6 @@ +pytalloc_CObject_FromTallocPtr: PyObject *(void *) +pytalloc_Check: int (PyObject *) +pytalloc_GetObjectType: PyTypeObject *(void) +pytalloc_reference_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) +pytalloc_steal: PyObject *(PyTypeObject *, void *) +pytalloc_steal_ex: PyObject *(PyTypeObject *, TALLOC_CTX *, void *) diff --git a/lib/talloc/ABI/talloc-2.0.9.sigs b/lib/talloc/ABI/talloc-2.0.9.sigs deleted file mode 100644 index eae12cc..0000000 --- a/lib/talloc/ABI/talloc-2.0.9.sigs +++ /dev/null @@ -1,64 +0,0 @@ -_talloc: void *(const void *, size_t) -_talloc_array: void *(const void *, size_t, unsigned int, const char *) -_talloc_free: int (void *, const char *) -_talloc_get_type_abort: void *(const void *, const char *, const char *) -_talloc_memdup: void *(const void *, const void *, size_t, const char *) -_talloc_move: void *(const void *, const void *) -_talloc_pooled_object: void *(const void *, size_t, const char *, unsigned int, size_t) -_talloc_realloc: void *(const void *, void *, size_t, const char *) -_talloc_realloc_array: void *(const void *, void *, size_t, unsigned int, const char *) -_talloc_reference_loc: void *(const void *, const void *, const char *) -_talloc_set_destructor: void (const void *, int (*)(void *)) -_talloc_steal_loc: void *(const void *, const void *, const char *) -_talloc_zero: void *(const void *, size_t, const char *) -_talloc_zero_array: void *(const void *, size_t, unsigned int, const char *) -talloc_asprintf: char *(const void *, const char *, ...) -talloc_asprintf_append: char *(char *, const char *, ...) -talloc_asprintf_append_buffer: char *(char *, const char *, ...) -talloc_autofree_context: void *(void) -talloc_check_name: void *(const void *, const char *) -talloc_disable_null_tracking: void (void) -talloc_enable_leak_report: void (void) -talloc_enable_leak_report_full: void (void) -talloc_enable_null_tracking: void (void) -talloc_enable_null_tracking_no_autofree: void (void) -talloc_find_parent_byname: void *(const void *, const char *) -talloc_free_children: void (void *) -talloc_get_name: const char *(const void *) -talloc_get_size: size_t (const void *) -talloc_increase_ref_count: int (const void *) -talloc_init: void *(const char *, ...) -talloc_is_parent: int (const void *, const void *) -talloc_named: void *(const void *, size_t, const char *, ...) -talloc_named_const: void *(const void *, size_t, const char *) -talloc_parent: void *(const void *) -talloc_parent_name: const char *(const void *) -talloc_pool: void *(const void *, size_t) -talloc_realloc_fn: void *(const void *, void *, size_t) -talloc_reference_count: size_t (const void *) -talloc_reparent: void *(const void *, const void *, const void *) -talloc_report: void (const void *, FILE *) -talloc_report_depth_cb: void (const void *, int, int, void (*)(const void *, int, int, int, void *), void *) -talloc_report_depth_file: void (const void *, int, int, FILE *) -talloc_report_full: void (const void *, FILE *) -talloc_set_abort_fn: void (void (*)(const char *)) -talloc_set_log_fn: void (void (*)(const char *)) -talloc_set_log_stderr: void (void) -talloc_set_memlimit: int (const void *, size_t) -talloc_set_name: const char *(const void *, const char *, ...) -talloc_set_name_const: void (const void *, const char *) -talloc_show_parents: void (const void *, FILE *) -talloc_strdup: char *(const void *, const char *) -talloc_strdup_append: char *(char *, const char *) -talloc_strdup_append_buffer: char *(char *, const char *) -talloc_strndup: char *(const void *, const char *, size_t) -talloc_strndup_append: char *(char *, const char *, size_t) -talloc_strndup_append_buffer: char *(char *, const char *, size_t) -talloc_total_blocks: size_t (const void *) -talloc_total_size: size_t (const void *) -talloc_unlink: int (const void *, void *) -talloc_vasprintf: char *(const void *, const char *, va_list) -talloc_vasprintf_append: char *(char *, const char *, va_list) -talloc_vasprintf_append_buffer: char *(char *, const char *, va_list) -talloc_version_major: int (void) -talloc_version_minor: int (void) diff --git a/lib/talloc/ABI/talloc-2.1.0.sigs b/lib/talloc/ABI/talloc-2.1.0.sigs new file mode 100644 index 0000000..eae12cc --- /dev/null +++ b/lib/talloc/ABI/talloc-2.1.0.sigs @@ -0,0 +1,64 @@ +_talloc: void *(const void *, size_t) +_talloc_array: void *(const void *, size_t, unsigned int, const char *) +_talloc_free: int (void *, const char *) +_talloc_get_type_abort: void *(const void *, const char *, const char *) +_talloc_memdup: void *(const void *, const void *, size_t, const char *) +_talloc_move: void *(const void *, const void *) +_talloc_pooled_object: void *(const void *, size_t, const char *, unsigned int, size_t) +_talloc_realloc: void *(const void *, void *, size_t, const char *) +_talloc_realloc_array: void *(const void *, void *, size_t, unsigned int, const char *) +_talloc_reference_loc: void *(const void *, const void *, const char *) +_talloc_set_destructor: void (const void *, int (*)(void *)) +_talloc_steal_loc: void *(const void *, const void *, const char *) +_talloc_zero: void *(const void *, size_t, const char *) +_talloc_zero_array: void *(const void *, size_t, unsigned int, const char *) +talloc_asprintf: char *(const void *, const char *, ...) +talloc_asprintf_append: char *(char *, const char *, ...) +talloc_asprintf_append_buffer: char *(char *, const char *, ...) +talloc_autofree_context: void *(void) +talloc_check_name: void *(const void *, const char *) +talloc_disable_null_tracking: void (void) +talloc_enable_leak_report: void (void) +talloc_enable_leak_report_full: void (void) +talloc_enable_null_tracking: void (void) +talloc_enable_null_tracking_no_autofree: void (void) +talloc_find_parent_byname: void *(const void *, const char *) +talloc_free_children: void (void *) +talloc_get_name: const char *(const void *) +talloc_get_size: size_t (const void *) +talloc_increase_ref_count: int (const void *) +talloc_init: void *(const char *, ...) +talloc_is_parent: int (const void *, const void *) +talloc_named: void *(const void *, size_t, const char *, ...) +talloc_named_const: void *(const void *, size_t, const char *) +talloc_parent: void *(const void *) +talloc_parent_name: const char *(const void *) +talloc_pool: void *(const void *, size_t) +talloc_realloc_fn: void *(const void *, void *, size_t) +talloc_reference_count: size_t (const void *) +talloc_reparent: void *(const void *, const void *, const void *) +talloc_report: void (const void *, FILE *) +talloc_report_depth_cb: void (const void *, int, int, void (*)(const void *, int, int, int, void *), void *) +talloc_report_depth_file: void (const void *, int, int, FILE *) +talloc_report_full: void (const void *, FILE *) +talloc_set_abort_fn: void (void (*)(const char *)) +talloc_set_log_fn: void (void (*)(const char *)) +talloc_set_log_stderr: void (void) +talloc_set_memlimit: int (const void *, size_t) +talloc_set_name: const char *(const void *, const char *, ...) +talloc_set_name_const: void (const void *, const char *) +talloc_show_parents: void (const void *, FILE *) +talloc_strdup: char *(const void *, const char *) +talloc_strdup_append: char *(char *, const char *) +talloc_strdup_append_buffer: char *(char *, const char *) +talloc_strndup: char *(const void *, const char *, size_t) +talloc_strndup_append: char *(char *, const char *, size_t) +talloc_strndup_append_buffer: char *(char *, const char *, size_t) +talloc_total_blocks: size_t (const void *) +talloc_total_size: size_t (const void *) +talloc_unlink: int (const void *, void *) +talloc_vasprintf: char *(const void *, const char *, va_list) +talloc_vasprintf_append: char *(char *, const char *, va_list) +talloc_vasprintf_append_buffer: char *(char *, const char *, va_list) +talloc_version_major: int (void) +talloc_version_minor: int (void) diff --git a/lib/talloc/talloc.c b/lib/talloc/talloc.c index 9229e9b..2824484 100644 --- a/lib/talloc/talloc.c +++ b/lib/talloc/talloc.c @@ -681,37 +681,39 @@ _PUBLIC_ void *talloc_pool(const void *context, size_t size) return result; } -_PUBLIC_ void *_talloc_pooled_object(const void *ctx, size_t size, - const char *name, - unsigned num_members, size_t member_size) +_PUBLIC_ void *_talloc_pooled_object(const void *ctx, + size_t type_size, + const char *type_name, + unsigned num_subobjects, + size_t total_subobjects_size) { - size_t poolsize, member_slack, tmp; + size_t poolsize, subobjects_slack, tmp; struct talloc_chunk *tc; struct talloc_pool_hdr *pool_hdr; void *ret; - poolsize = size + member_size; + poolsize = type_size + total_subobjects_size; - if ((poolsize < size) || (poolsize < member_size)) { + if ((poolsize < type_size) || (poolsize < total_subobjects_size)) { goto overflow; } - if (num_members == UINT_MAX) { + if (num_subobjects == UINT_MAX) { goto overflow; } - num_members += 1; /* the object body itself */ + num_subobjects += 1; /* the object body itself */ /* * Alignment can increase the pool size by at most 15 bytes per object * plus alignment for the object itself */ - member_slack = (TC_HDR_SIZE + TP_HDR_SIZE + 15) * num_members; - if (member_slack < num_members) { + subobjects_slack = (TC_HDR_SIZE + TP_HDR_SIZE + 15) * num_subobjects; + if (subobjects_slack < num_subobjects) { goto overflow; } - tmp = poolsize + member_slack; - if ((tmp < poolsize) || (tmp < member_slack)) { + tmp = poolsize + subobjects_slack; + if ((tmp < poolsize) || (tmp < subobjects_slack)) { goto overflow; } poolsize = tmp; @@ -722,17 +724,17 @@ _PUBLIC_ void *_talloc_pooled_object(const void *ctx, size_t size, } tc = talloc_chunk_from_ptr(ret); - tc->size = size; + tc->size = type_size; pool_hdr = talloc_pool_from_chunk(tc); #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) - VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, size); + VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, type_size); #endif - pool_hdr->end = ((char *)pool_hdr->end + TC_ALIGN16(size)); + pool_hdr->end = ((char *)pool_hdr->end + TC_ALIGN16(type_size)); - talloc_set_name_const(ret, name); + talloc_set_name_const(ret, type_name); return ret; overflow: diff --git a/lib/talloc/talloc.h b/lib/talloc/talloc.h index 5e27484..2b46b75 100644 --- a/lib/talloc/talloc.h +++ b/lib/talloc/talloc.h @@ -848,14 +848,40 @@ void *talloc_find_parent_bytype(const void *ptr, #type); void *talloc_pool(const void *context, size_t size); #ifdef DOXYGEN +/** + * @brief Allocate a talloc object as/with an additional pool. + * + * This is like talloc_pool(), but's it's more flexible + * and allows an object to be a pool for its children. + * + * @param[in] ctx The talloc context to hang the result off. + * + * @param[in] type The type that we want to allocate. + * + * @param[in] num_subobjects The expected number of subobjects, which will + * be allocated within the pool. This allocates + * space for talloc_chunk headers. + * + * @param[in] total_subobjects_size The size that all subobjects can use in total. + * + * + * @return The allocated talloc object, NULL on error. + */ void *talloc_pooled_object(const void *ctx, #type, - unsigned num_members, size_t member_size); + unsigned num_subobjects, + size_t total_subobjects_size); #else -#define talloc_pooled_object(_ctx, _type, _num_members, _member_size) \ +#define talloc_pooled_object(_ctx, _type, \ + _num_subobjects, \ + _total_subobjects_size) \ (_type *)_talloc_pooled_object((_ctx), sizeof(_type), #_type, \ - (_num_members), (_member_size)) -void *_talloc_pooled_object(const void *ctx, size_t size, const char *name, - unsigned num_members, size_t member_size); + (_num_subobjects), \ + (_total_subobjects_size)) +void *_talloc_pooled_object(const void *ctx, + size_t type_size, + const char *type_name, + unsigned num_subobjects, + size_t total_subobjects_size); #endif /** diff --git a/lib/talloc/wscript b/lib/talloc/wscript index 8c13b1d..1ca41f6 100644 --- a/lib/talloc/wscript +++ b/lib/talloc/wscript @@ -1,7 +1,7 @@ #!/usr/bin/env python APPNAME = 'talloc' -VERSION = '2.0.9' +VERSION = '2.1.0' blddir = 'bin' -- 1.7.9.5 From 311dc3d9a436ce7bb14395d1b764ac9aa7ddd55b Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Sun, 12 May 2013 09:22:23 +0200 Subject: [PATCH 10/12] talloc: Test the pooled object Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/talloc/testsuite.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/talloc/testsuite.c b/lib/talloc/testsuite.c index 5501d17..be8f076 100644 --- a/lib/talloc/testsuite.c +++ b/lib/talloc/testsuite.c @@ -1291,6 +1291,40 @@ static bool test_pool_nest(void) return true; } +struct pooled { + char *s1; + char *s2; + char *s3; +}; + +static bool test_pooled_object(void) +{ + struct pooled *p; + const char *s1 = "hello"; + const char *s2 = "world"; + const char *s3 = ""; + + p = talloc_pooled_object(NULL, struct pooled, 3, + strlen(s1)+strlen(s2)+strlen(s3)+3); + + if (talloc_get_size(p) != sizeof(struct pooled)) { + return false; + } + + p->s1 = talloc_strdup(p, s1); + + TALLOC_FREE(p->s1); + p->s1 = talloc_strdup(p, s2); + TALLOC_FREE(p->s1); + + p->s1 = talloc_strdup(p, s1); + p->s2 = talloc_strdup(p, s2); + p->s3 = talloc_strdup(p, s3); + + TALLOC_FREE(p); + return true; +} + static bool test_free_ref_null_context(void) { void *p1, *p2, *p3; @@ -1564,6 +1598,8 @@ bool torture_local_talloc(struct torture_context *tctx) setlinebuf(stdout); test_reset(); + ret &= test_pooled_object(); + test_reset(); ret &= test_pool_nest(); test_reset(); ret &= test_ref1(); -- 1.7.9.5 From c799346d8763857a79b5262ba78f0988f0173a26 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 9 May 2013 13:55:08 +0200 Subject: [PATCH 11/12] TODO-FIX: smbd: Use talloc_pooled_object in cp_smb_filename Requires new talloc Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: talloc_zero => ZERO_STRUCT??? --- source3/lib/filename_util.c | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/source3/lib/filename_util.c b/source3/lib/filename_util.c index 19ffcc3..ace09d9 100644 --- a/source3/lib/filename_util.c +++ b/source3/lib/filename_util.c @@ -161,39 +161,49 @@ struct smb_filename *cp_smb_filename(TALLOC_CTX *mem_ctx, const struct smb_filename *in) { struct smb_filename *out; + size_t base_len = 0; + size_t stream_len = 0; + size_t lcomp_len = 0; + int num = 0; /* stream_name must always be NULL if there is no stream. */ if (in->stream_name) { SMB_ASSERT(in->stream_name[0] != '\0'); } - out = talloc_zero(mem_ctx, struct smb_filename); + if (in->base_name != NULL) { + base_len = strlen(in->base_name) + 1; + num += 1; + } + if (in->stream_name != NULL) { + stream_len = strlen(in->stream_name) + 1; + num += 1; + } + if (in->original_lcomp != NULL) { + lcomp_len = strlen(in->original_lcomp) + 1; + num += 1; + } + + out = talloc_pooled_object(mem_ctx, struct smb_filename, + num, stream_len + base_len + lcomp_len); if (out == NULL) { return NULL; } + if (in->base_name != NULL) { - out->base_name = talloc_strdup(out, in->base_name); - if (out->base_name == NULL) { - goto fail; - } + out->base_name = talloc_memdup( + out, in->base_name, base_len); } if (in->stream_name != NULL) { - out->stream_name = talloc_strdup(out, in->stream_name); - if (out->stream_name == NULL) { - goto fail; - } + out->stream_name = talloc_memdup( + out, in->stream_name, stream_len); } if (in->original_lcomp != NULL) { - out->original_lcomp = talloc_strdup(out, in->original_lcomp); - if (out->original_lcomp == NULL) { - goto fail; - } + out->original_lcomp = talloc_memdup( + out, in->original_lcomp, lcomp_len); } out->st = in->st; return out; -fail: - TALLOC_FREE(out); - return NULL; } /**************************************************************************** -- 1.7.9.5 From 83c4879c195e492e8101b57c4ab974975ee5b582 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Sat, 3 Aug 2013 15:49:59 +0200 Subject: [PATCH 12/12] TODO-FIX: tevent: Use talloc_pooled_object for tevent_req_create Signed-off-by: Volker Lendecke Reviewed-by: Jeremy Allison TODO: talloc_zero => ZERO_STRUCT??? --- lib/tevent/tevent_req.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tevent/tevent_req.c b/lib/tevent/tevent_req.c index d8d0c5f..a280dd3 100644 --- a/lib/tevent/tevent_req.c +++ b/lib/tevent/tevent_req.c @@ -61,7 +61,9 @@ struct tevent_req *_tevent_req_create(TALLOC_CTX *mem_ctx, void **ppdata = (void **)pdata; void *data; - req = talloc_zero(mem_ctx, struct tevent_req); + req = talloc_pooled_object( + mem_ctx, struct tevent_req, 2, + sizeof(struct tevent_immediate) + data_size); if (req == NULL) { return NULL; } -- 1.7.9.5