performance with >50GB files

[René Rebe]

Hi,

On Wednesday 11 January 2006 00:31, Wayne Davison wrote:

> So, perhaps the size of the sending file should be factored into the
> calculation in order to set a minimum acceptable block size.  This would
> be easy, because the generator already knows the size of both files at
> the time that it performs this calculation (or else it wouldn't have
> been able to figure out if the file was up-to-date or not).

So, here is some hackery to scale the block size from 700 bytes for a near
zero sized file to 5MB for a 50GB file and linear increasing using the bigger size
of the the two available sources.

But I fear you want to keep something like the old, rather overly complex IMHO
block size algorithm but tweak some factor in it ...

--- rsync-2.6.6/generator.c	2005-07-28 21:06:03.000000000 +0200
+++ rsync-2.6.6-fixed/generator.c	2006-01-11 12:34:01.000000000 +0100
@@ -405,25 +405,11 @@
 
 	if (block_size)
 		blength = block_size;
-	else if (len <= BLOCK_SIZE * BLOCK_SIZE)
-		blength = BLOCK_SIZE;
 	else {
-		int32 c;
-		int64 l;
-		int cnt;
-		for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
-		if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
+		blength = len / 10000 + BLOCK_SIZE;
+		blength -= blength % 8; /* round to multiple of 8 */
+		if (blength >= MAX_BLOCK_SIZE)
 			blength = MAX_BLOCK_SIZE;
-		else {
-		    blength = 0;
-		    do {
-			    blength |= c;
-			    if (len < (int64)blength * blength)
-				    blength &= ~c;
-			    c >>= 1;
-		    } while (c >= 8);	/* round to multiple of 8 */
-		    blength = MAX(blength, BLOCK_SIZE);
-		}
 	}
 
 	if (protocol_version < 27) {
@@ -462,14 +448,14 @@
  *
  * Generate approximately one checksum every block_len bytes.
  */
-static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
+static void generate_and_send_sums(int fd, OFF_T len, OFF_T len2, int f_out, int f_copy)
 {
 	int32 i;
 	struct map_struct *mapbuf;
 	struct sum_struct sum;
 	OFF_T offset = 0;
 
-	sum_sizes_sqroot(&sum, len);
+	sum_sizes_sqroot(&sum, MAX(len, len2));
 
 	if (len > 0)
 		mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
@@ -1123,7 +1109,7 @@
 		return;
 	}
 
-	generate_and_send_sums(fd, st.st_size, f_out, f_copy);
+	generate_and_send_sums(fd, st.st_size, file->length, f_out, f_copy);
 
 	if (f_copy >= 0) {
 		close(f_copy);


-- 
René Rebe - Rubensstr. 64 - 12157 Berlin (Europe / Germany)
            http://www.exactcode.de | http://www.t2-project.org
            +49 (0)30  255 897 45