[PATCH 1/2] tdb: don't corrupt database if we go overlength due to transaction expand.

Rusty Russell rusty at rustcorp.com.au
Sun Jun 2 18:37:41 MDT 2013


Volker Lendecke <Volker.Lendecke at SerNet.DE> writes:
> On Wed, May 29, 2013 at 10:33:16AM +0930, Rusty Russell wrote:
>> I think I prefer a bool return here when we fail.  That has the nice
>> property of doing our overflow checking in one place.
>> 
>> Hmm, not quite: tdb_recovery_size() could theoretically overflow.  So
>> could the addition of sizeof(rec).  I think a bool tdb_overflow() helper
>> might be a good idea:
>> 
>>         bool tdb_overflow(tdb_off_t *ret, tdb_off_t a, tdb_off_t b)
>>         {
>>                 if (a + b < a)
>>                         return false;
>>                 *ret = a + b;
>>                 return true;
>>         }
>> 
>> We can then make tdb_recovery_size() return a bool, too...
>
> Attached find a new patchset. I did not change the
> tdb_expand_adjust return value, I think this is more than we
> want to address here. But I did add the overflow-checking
> add function and used that where I think it's appropriate.
>
> Haven't added your Reviewed-by yet, the patches now slightly
> differ due to the tdb_add_off_t routine.
>
> Volker
>
> P.S: To me it seems we're implementing Ada integer semantics
> here....

Yes, limiting overflow.  And it's my fault, since we used to be 31 bit
limited and didn't have any of these problems.

Reviewed-by: Rusty Russell <rusty at rustcorp.com.au>

Thanks,
Rusty.

> -- 
> SerNet GmbH, Bahnhofsallee 1b, 37081 Göttingen
> phone: +49-551-370000-0, fax: +49-551-370000-9
> AG Göttingen, HRB 2816, GF: Dr. Johannes Loxen
> http://www.sernet.de, mailto:kontakt at sernet.de
> From 3a50fc70d3b7004bd4624ad5f5639e2d429891d7 Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 30 May 2013 14:52:59 +0200
> Subject: [PATCH 1/8] tdb: Add overflow-checking tdb_add_off_t
>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
>  lib/tdb/common/tdb.c         |   11 +++++++++++
>  lib/tdb/common/tdb_private.h |    1 +
>  2 files changed, 12 insertions(+)
>
> diff --git a/lib/tdb/common/tdb.c b/lib/tdb/common/tdb.c
> index a2ae187..6256a05 100644
> --- a/lib/tdb/common/tdb.c
> +++ b/lib/tdb/common/tdb.c
> @@ -1000,6 +1000,17 @@ bool tdb_write_all(int fd, const void *buf, size_t count)
>  	return true;
>  }
>  
> +bool tdb_add_off_t(tdb_off_t a, tdb_off_t b, tdb_off_t *pret)
> +{
> +	tdb_off_t ret = a + b;
> +
> +	if ((ret < a) || (ret < b)) {
> +		return false;
> +	}
> +	*pret = ret;
> +	return true;
> +}
> +
>  #ifdef TDB_TRACE
>  static void tdb_trace_write(struct tdb_context *tdb, const char *str)
>  {
> diff --git a/lib/tdb/common/tdb_private.h b/lib/tdb/common/tdb_private.h
> index 406fc5f..c37246f 100644
> --- a/lib/tdb/common/tdb_private.h
> +++ b/lib/tdb/common/tdb_private.h
> @@ -282,4 +282,5 @@ void tdb_header_hash(struct tdb_context *tdb,
>  		     uint32_t *magic1_hash, uint32_t *magic2_hash);
>  unsigned int tdb_old_hash(TDB_DATA *key);
>  size_t tdb_dead_space(struct tdb_context *tdb, tdb_off_t off);
> +bool tdb_add_off_t(tdb_off_t a, tdb_off_t b, tdb_off_t *pret);
>  #endif /* TDB_PRIVATE_H */
> -- 
> 1.7.9.5
>
>
> From 44d47c39ff5632582a4dd08bddc0ce6eeb4d08c6 Mon Sep 17 00:00:00 2001
> From: Stefan Metzmacher <metze at samba.org>
> Date: Tue, 28 May 2013 12:56:57 +0200
> Subject: [PATCH 2/8] tdb: add a 'new_size' helper variable to
>  tdb_expand_file()
>
> Pair-Programmed-With: Volker Lendecke <vl at samba.org>
>
> Signed-off-by: Stefan Metzmacher <metze at samba.org>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
>  lib/tdb/common/io.c |   11 +++++++----
>  1 file changed, 7 insertions(+), 4 deletions(-)
>
> diff --git a/lib/tdb/common/io.c b/lib/tdb/common/io.c
> index a477fb5..44ef728 100644
> --- a/lib/tdb/common/io.c
> +++ b/lib/tdb/common/io.c
> @@ -287,18 +287,21 @@ int tdb_mmap(struct tdb_context *tdb)
>  static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition)
>  {
>  	char buf[8192];
> +	tdb_off_t new_size;
>  
>  	if (tdb->read_only || tdb->traverse_read) {
>  		tdb->ecode = TDB_ERR_RDONLY;
>  		return -1;
>  	}
>  
> -	if (ftruncate(tdb->fd, size+addition) == -1) {
> +	new_size = size + addition;
> +
> +	if (ftruncate(tdb->fd, new_size) == -1) {
>  		char b = 0;
> -		ssize_t written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
> +		ssize_t written = pwrite(tdb->fd,  &b, 1, new_size - 1);
>  		if (written == 0) {
>  			/* try once more, potentially revealing errno */
> -			written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
> +			written = pwrite(tdb->fd,  &b, 1, new_size - 1);
>  		}
>  		if (written == 0) {
>  			/* again - give up, guessing errno */
> @@ -306,7 +309,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
>  		}
>  		if (written != 1) {
>  			TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %u failed (%s)\n",
> -				 size+addition, strerror(errno)));
> +				 (unsigned)new_size, strerror(errno)));
>  			return -1;
>  		}
>  	}
> -- 
> 1.7.9.5
>
>
> From e90e7c78f2fe6a003960c714d2864787864ba30b Mon Sep 17 00:00:00 2001
> From: Stefan Metzmacher <metze at samba.org>
> Date: Tue, 28 May 2013 12:59:32 +0200
> Subject: [PATCH 3/8] tdb: add overflow/ENOSPC handling to tdb_expand_file()
>
> Pair-Programmed-With: Volker Lendecke <vl at samba.org>
>
> Signed-off-by: Stefan Metzmacher <metze at samba.org>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
>  lib/tdb/common/io.c |   12 +++++++++++-
>  1 file changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/lib/tdb/common/io.c b/lib/tdb/common/io.c
> index 44ef728..d177640 100644
> --- a/lib/tdb/common/io.c
> +++ b/lib/tdb/common/io.c
> @@ -294,7 +294,14 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
>  		return -1;
>  	}
>  
> -	new_size = size + addition;
> +	if (!tdb_add_off_t(size, addition, &new_size)) {
> +		tdb->ecode = TDB_ERR_OOM;
> +		TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
> +			"overflow detected current size[%u] addition[%u]!\n",
> +			(unsigned)size, (unsigned)addition));
> +		errno = ENOSPC;
> +		return -1;
> +	}
>  
>  	if (ftruncate(tdb->fd, new_size) == -1) {
>  		char b = 0;
> @@ -308,6 +315,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
>  			errno = ENOSPC;
>  		}
>  		if (written != 1) {
> +			tdb->ecode = TDB_ERR_OOM;
>  			TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %u failed (%s)\n",
>  				 (unsigned)new_size, strerror(errno)));
>  			return -1;
> @@ -327,12 +335,14 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
>  		}
>  		if (written == 0) {
>  			/* give up, trying to provide a useful errno */
> +			tdb->ecode = TDB_ERR_OOM;
>  			TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write "
>  				"returned 0 twice: giving up!\n"));
>  			errno = ENOSPC;
>  			return -1;
>  		}
>  		if (written == -1) {
> +			tdb->ecode = TDB_ERR_OOM;
>  			TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of "
>  				 "%u bytes failed (%s)\n", (int)n,
>  				 strerror(errno)));
> -- 
> 1.7.9.5
>
>
> From d724172639056fafa41ca669887d2dbcbf62075f Mon Sep 17 00:00:00 2001
> From: Stefan Metzmacher <metze at samba.org>
> Date: Tue, 28 May 2013 13:01:27 +0200
> Subject: [PATCH 4/8] tdb: add overflow detection to tdb_expand_adjust()
>
> We round up at maximun to a new size of 4GB,
> but still return at least the given size.
>
> The caller has to deal with ENOSPC itself.
>
> Pair-Programmed-With: Volker Lendecke <vl at samba.org>
>
> Signed-off-by: Stefan Metzmacher <metze at samba.org>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
>  lib/tdb/common/io.c |   34 +++++++++++++++++++++++++++++++---
>  1 file changed, 31 insertions(+), 3 deletions(-)
>
> diff --git a/lib/tdb/common/io.c b/lib/tdb/common/io.c
> index d177640..c9c9fa8 100644
> --- a/lib/tdb/common/io.c
> +++ b/lib/tdb/common/io.c
> @@ -363,14 +363,32 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
>  /* You need 'size', this tells you how much you should expand by. */
>  tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
>  {
> -	tdb_off_t new_size, top_size;
> +	tdb_off_t new_size, top_size, increment;
> +	tdb_off_t max_size = UINT32_MAX - map_size;
> +
> +	if (size > max_size) {
> +		/*
> +		 * We can't round up anymore, just give back
> +		 * what we're asked for.
> +		 *
> +		 * The caller has to take care of the ENOSPC handling.
> +		 */
> +		return size;
> +	}
>  
>  	/* limit size in order to avoid using up huge amounts of memory for
>  	 * in memory tdbs if an oddball huge record creeps in */
>  	if (size > 100 * 1024) {
> -		top_size = map_size + size * 2;
> +		increment = size * 2;
>  	} else {
> -		top_size = map_size + size * 100;
> +		increment = size * 100;
> +	}
> +	if (increment < size) {
> +		goto overflow;
> +	}
> +
> +	if (!tdb_add_off_t(map_size, increment, &top_size)) {
> +		goto overflow;
>  	}
>  
>  	/* always make room for at least top_size more records, and at
> @@ -381,10 +399,20 @@ tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
>  	} else {
>  		new_size = map_size * 1.25;
>  	}
> +	if (new_size < map_size) {
> +		goto overflow;
> +	}
>  
>  	/* Round the database up to a multiple of the page size */
>  	new_size = MAX(top_size, new_size);
>  	return TDB_ALIGN(new_size, page_size) - map_size;
> +
> +overflow:
> +	/*
> +	 * Somewhere in between we went over 4GB. Make one big jump to
> +	 * exactly 4GB database size.
> +	 */
> +	return max_size;
>  }
>  
>  /* expand the database at least size bytes by expanding the underlying
> -- 
> 1.7.9.5
>
>
> From 8b5dda2d01e85fa9b96945417956272107b8b193 Mon Sep 17 00:00:00 2001
> From: Stefan Metzmacher <metze at samba.org>
> Date: Tue, 28 May 2013 13:04:29 +0200
> Subject: [PATCH 5/8] tdb: add proper OOM/ENOSPC handling to tdb_expand()
>
> Failing to do so will result in corrupt tdbs: We will overwrite
> the hash chain pointers with 0x42424242.
>
> Pair-Programmed-With: Volker Lendecke <vl at samba.org>
>
> Signed-off-by: Stefan Metzmacher <metze at samba.org>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
>  lib/tdb/common/io.c |   31 +++++++++++++++++++++++--------
>  1 file changed, 23 insertions(+), 8 deletions(-)
>
> diff --git a/lib/tdb/common/io.c b/lib/tdb/common/io.c
> index c9c9fa8..87d47b9 100644
> --- a/lib/tdb/common/io.c
> +++ b/lib/tdb/common/io.c
> @@ -421,6 +421,7 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
>  {
>  	struct tdb_record rec;
>  	tdb_off_t offset;
> +	tdb_off_t new_size;
>  
>  	if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
>  		TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
> @@ -432,10 +433,12 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
>  
>  	size = tdb_expand_adjust(tdb->map_size, size, tdb->page_size);
>  
> -	/* expand the file itself */
> -	if (!(tdb->flags & TDB_INTERNAL)) {
> -		if (tdb->methods->tdb_expand_file(tdb, tdb->map_size, size) != 0)
> -			goto fail;
> +	if (!tdb_add_off_t(tdb->map_size, size, &new_size)) {
> +		tdb->ecode = TDB_ERR_OOM;
> +		TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_expand "
> +			"overflow detected current map_size[%u] size[%u]!\n",
> +			(unsigned)tdb->map_size, (unsigned)size));
> +		goto fail;
>  	}
>  
>  	/* form a new freelist record */
> @@ -444,18 +447,30 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
>  	rec.rec_len = size - sizeof(rec);
>  
>  	if (tdb->flags & TDB_INTERNAL) {
> -		char *new_map_ptr = (char *)realloc(tdb->map_ptr,
> -						    tdb->map_size + size);
> +		char *new_map_ptr;
> +
> +		new_map_ptr = (char *)realloc(tdb->map_ptr, new_size);
>  		if (!new_map_ptr) {
> +			tdb->ecode = TDB_ERR_OOM;
>  			goto fail;
>  		}
>  		tdb->map_ptr = new_map_ptr;
> -		tdb->map_size += size;
> +		tdb->map_size = new_size;
>  	} else {
> +		int ret;
> +
> +		/*
> +		 * expand the file itself
> +		 */
> +		ret = tdb->methods->tdb_expand_file(tdb, tdb->map_size, size);
> +		if (ret != 0) {
> +			goto fail;
> +		}
> +
>  		/* Explicitly remap: if we're in a transaction, this won't
>  		 * happen automatically! */
>  		tdb_munmap(tdb);
> -		tdb->map_size += size;
> +		tdb->map_size = new_size;
>  		if (tdb_mmap(tdb) != 0) {
>  			goto fail;
>  		}
> -- 
> 1.7.9.5
>
>
> From 36c922d2387f354ee2276ec67dbb300e58d04e78 Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 30 May 2013 15:24:27 +0200
> Subject: [PATCH 6/8] tdb: Make tdb_recovery_size overflow-safe
>
> ---
>  lib/tdb/common/tdb_private.h |    3 +++
>  lib/tdb/common/transaction.c |   32 +++++++++++++++++++++++++-------
>  2 files changed, 28 insertions(+), 7 deletions(-)
>
> diff --git a/lib/tdb/common/tdb_private.h b/lib/tdb/common/tdb_private.h
> index c37246f..ce92188 100644
> --- a/lib/tdb/common/tdb_private.h
> +++ b/lib/tdb/common/tdb_private.h
> @@ -283,4 +283,7 @@ void tdb_header_hash(struct tdb_context *tdb,
>  unsigned int tdb_old_hash(TDB_DATA *key);
>  size_t tdb_dead_space(struct tdb_context *tdb, tdb_off_t off);
>  bool tdb_add_off_t(tdb_off_t a, tdb_off_t b, tdb_off_t *pret);
> +
> +/* tdb_off_t and tdb_len_t right now are both uint32_t */
> +#define tdb_add_len_t tdb_add_off_t
>  #endif /* TDB_PRIVATE_H */
> diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
> index 81cfd16..080d058 100644
> --- a/lib/tdb/common/transaction.c
> +++ b/lib/tdb/common/transaction.c
> @@ -630,28 +630,37 @@ _PUBLIC_ int tdb_transaction_cancel(struct tdb_context *tdb)
>  /*
>    work out how much space the linearised recovery data will consume
>  */
> -static tdb_len_t tdb_recovery_size(struct tdb_context *tdb)
> +static bool tdb_recovery_size(struct tdb_context *tdb, tdb_len_t *result)
>  {
>  	tdb_len_t recovery_size = 0;
>  	int i;
>  
>  	recovery_size = sizeof(uint32_t);
>  	for (i=0;i<tdb->transaction->num_blocks;i++) {
> +		tdb_len_t block_size;
>  		if (i * tdb->transaction->block_size >= tdb->transaction->old_map_size) {
>  			break;
>  		}
>  		if (tdb->transaction->blocks[i] == NULL) {
>  			continue;
>  		}
> -		recovery_size += 2*sizeof(tdb_off_t);
> +		if (!tdb_add_len_t(recovery_size, 2*sizeof(tdb_off_t),
> +				   &recovery_size)) {
> +			return false;
> +		}
>  		if (i == tdb->transaction->num_blocks-1) {
> -			recovery_size += tdb->transaction->last_block_size;
> +			block_size = tdb->transaction->last_block_size;
>  		} else {
> -			recovery_size += tdb->transaction->block_size;
> +			block_size =  tdb->transaction->block_size;
> +		}
> +		if (!tdb_add_len_t(recovery_size, block_size,
> +				   &recovery_size)) {
> +			return false;
>  		}
>  	}
>  
> -	return recovery_size;
> +	*result = recovery_size;
> +	return true;
>  }
>  
>  int tdb_recovery_area(struct tdb_context *tdb,
> @@ -700,7 +709,11 @@ static int tdb_recovery_allocate(struct tdb_context *tdb,
>  		return -1;
>  	}
>  
> -	*recovery_size = tdb_recovery_size(tdb);
> +	if (!tdb_recovery_size(tdb, recovery_size)) {
> +		TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_recovery_allocate: "
> +			 "overflow recovery size\n"));
> +		return -1;
> +	}
>  
>  	/* Existing recovery area? */
>  	if (recovery_head != 0 && *recovery_size <= rec.rec_len) {
> @@ -728,7 +741,12 @@ static int tdb_recovery_allocate(struct tdb_context *tdb,
>  
>  			/* the tdb_free() call might have increased
>  			 * the recovery size */
> -			*recovery_size = tdb_recovery_size(tdb);
> +			if (!tdb_recovery_size(tdb, recovery_size)) {
> +				TDB_LOG((tdb, TDB_DEBUG_FATAL,
> +					 "tdb_recovery_allocate: "
> +					 "overflow recovery size\n"));
> +				return -1;
> +			}
>  		}
>  
>  		/* New head will be at end of file. */
> -- 
> 1.7.9.5
>
>
> From db315ae5ad416d008b040b94ac45f7f9be4ca6ad Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 30 May 2013 15:54:58 +0200
> Subject: [PATCH 7/8] tdb: Make tdb_recovery_allocate overflow-safe
>
> ---
>  lib/tdb/common/transaction.c |    7 ++++++-
>  1 file changed, 6 insertions(+), 1 deletion(-)
>
> diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
> index 080d058..a2c3bbd 100644
> --- a/lib/tdb/common/transaction.c
> +++ b/lib/tdb/common/transaction.c
> @@ -762,7 +762,12 @@ static int tdb_recovery_allocate(struct tdb_context *tdb,
>  					       tdb->page_size)
>  		- sizeof(rec);
>  
> -	new_end = recovery_head + sizeof(rec) + *recovery_max_size;
> +	if (!tdb_add_off_t(recovery_head, sizeof(rec), &new_end) ||
> +	    !tdb_add_off_t(new_end, *recovery_max_size, &new_end)) {
> +		TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_recovery_allocate: "
> +			 "overflow recovery area\n"));
> +		return -1;
> +	}
>  
>  	if (methods->tdb_expand_file(tdb, tdb->transaction->old_map_size,
>  				     new_end - tdb->transaction->old_map_size)
> -- 
> 1.7.9.5
>
>
> From f77dee1264460ea4426f6602b4d4924d300ec077 Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 30 May 2013 16:23:17 +0200
> Subject: [PATCH 8/8] tdb: Add another overflow check to tdb_expand_adjust
>
> ---
>  lib/tdb/common/io.c |    6 ++++++
>  1 file changed, 6 insertions(+)
>
> diff --git a/lib/tdb/common/io.c b/lib/tdb/common/io.c
> index 87d47b9..11dfefd 100644
> --- a/lib/tdb/common/io.c
> +++ b/lib/tdb/common/io.c
> @@ -405,6 +405,12 @@ tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
>  
>  	/* Round the database up to a multiple of the page size */
>  	new_size = MAX(top_size, new_size);
> +
> +	if (new_size + page_size < new_size) {
> +		/* There's a "+" in TDB_ALIGN that might overflow... */
> +		goto overflow;
> +	}
> +
>  	return TDB_ALIGN(new_size, page_size) - map_size;
>  
>  overflow:
> -- 
> 1.7.9.5


More information about the samba-technical mailing list