MD4 checksum fix

[cbarratt at users.sourceforge.net]

> > I agree, they should be done together.  I don't have my original
> > patch but I can reimplement it with the correct remote_version
> > dependence and send it in the next couple of days (by Thursday
> > evening).  My intent is the minimal set of changes, rather than
> > changing the internals.
> 
> That is a workable timeframe, thanks.

Two days late, but here is the patch against 2.5.6.  This implements the
following changes:

 - for protocol version >= 27, mdfour_tail() is called when the block size
   (including checksum_seed) is a multiple of 64.  Previously it was not
   called, giving the wrong MD4 checksum.

 - for protocol version >= 27, a 64 bit bit counter is used in mdfour.c as
   required by the RFC.  Previously only a 32 bit bit counter was used,
   causing incorrect MD4 file checksums for file sizes >= 512MB - 4.

Craig

###########################################################################
diff -bur rsync-2.5.6/checksum.c rsync-2.5.6-fixed-md4/checksum.c
--- rsync-2.5.6/checksum.c	2002-04-08 01:29:04.000000000 -0700
+++ rsync-2.5.6-fixed-md4/checksum.c	2003-04-05 16:03:15.000000000 -0800
@@ -74,7 +74,13 @@
 	for(i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
 		mdfour_update(&m, (uchar *)(buf1+i), CSUM_CHUNK);
 	}
-	if (len - i > 0) {
+	/*
+	 * Prior to version 27 an incorrect MD4 checksum was computed
+	 * by failing to call mdfour_tail() for block sizes that
+	 * are multiples of 64.  This is fixed by calling mdfour_update()
+	 * even when there are no more bytes.
+	 */
+	if (len - i > 0 || (remote_version >= 27 && len - i == 0)) {
 		mdfour_update(&m, (uchar *)(buf1+i), (len-i));
 	}
 	
@@ -105,8 +111,16 @@
 		mdfour_update(&m, (uchar *)tmpchunk, CSUM_CHUNK);
 	}
 
+	/*
+	 * Prior to version 27 an incorrect MD4 checksum was computed
+	 * by failing to call mdfour_tail() for block sizes that
+	 * are multiples of 64.  This is fixed by calling mdfour_update()
+	 * even when there are no more bytes.
+	 */
+	if (len - i > 0 || (remote_version >= 27 && len - i == 0)) {
 	if (len - i > 0) {
 		memcpy(tmpchunk, map_ptr(buf,i,len-i), len-i);
+		}
 		mdfour_update(&m, (uchar *)tmpchunk, (len-i));
 	}
 
diff -bur rsync-2.5.6/lib/mdfour.c rsync-2.5.6-fixed-md4/lib/mdfour.c
--- rsync-2.5.6/lib/mdfour.c	2001-06-27 22:07:15.000000000 -0700
+++ rsync-2.5.6-fixed-md4/lib/mdfour.c	2003-04-05 16:04:46.000000000 -0800
@@ -109,29 +109,53 @@
 	md->C = 0x98badcfe;
 	md->D = 0x10325476;
 	md->totalN = 0;
+	md->totalN2 = 0;
 }
 
 
-static void mdfour_tail(unsigned char *in, int n)
+static void mdfour_tail(unsigned char *in, uint32 n)
 {
 	unsigned char buf[128];
 	uint32 M[16];
-	uint32 b;
+	extern int remote_version;
 
-	m->totalN += n;
-
-	b = m->totalN * 8;
+	/*
+	 * Count total number of bits, modulo 2^64
+	 */
+	m->totalN += n << 3;
+	if (m->totalN < (n << 3)) {
+		m->totalN2++;
+	}
+	m->totalN2 += n >> 29;
 
 	memset(buf, 0, 128);
 	if (n) memcpy(buf, in, n);
 	buf[n] = 0x80;
 
 	if (n <= 55) {
-		copy4(buf+56, b);
+		copy4(buf+56, m->totalN);
+		/*
+		 * Prior to protocol version 27 only the number of bits
+		 * modulo 2^32 was included.  MD4 requires the number
+		 * of bits modulo 2^64, which was fixed starting with
+		 * protocol version 27.
+		 */
+		if (remote_version >= 27) {
+			copy4(buf+60, m->totalN2);
+		}
 		copy64(M, buf);
 		mdfour64(M);
 	} else {
-		copy4(buf+120, b); 
+		copy4(buf+120, m->totalN); 
+		/*
+		 * Prior to protocol version 27 only the number of bits
+		 * modulo 2^32 was included.  MD4 requires the number
+		 * of bits modulo 2^64, which was fixed starting with
+		 * protocol version 27.
+		 */
+		if (remote_version >= 27) {
+			copy4(buf+124, m->totalN2); 
+		}
 		copy64(M, buf);
 		mdfour64(M);
 		copy64(M, buf+64);
@@ -139,20 +163,23 @@
 	}
 }
 
-void mdfour_update(struct mdfour *md, unsigned char *in, int n)
+void mdfour_update(struct mdfour *md, unsigned char *in, uint32 n)
 {
 	uint32 M[16];
 
-	if (n == 0) mdfour_tail(in, n);
-
 	m = md;
 
+	if (n == 0) mdfour_tail(in, n);
+
 	while (n >= 64) {
 		copy64(M, in);
 		mdfour64(M);
 		in += 64;
 		n -= 64;
-		m->totalN += 64;
+		m->totalN += 64 << 3;
+		if (m->totalN < 64 << 3) {
+			m->totalN2++;
+		}
 	}
 
 	if (n) mdfour_tail(in, n);
diff -bur rsync-2.5.6/lib/mdfour.h rsync-2.5.6-fixed-md4/lib/mdfour.h
--- rsync-2.5.6/lib/mdfour.h	1998-10-28 21:01:52.000000000 -0800
+++ rsync-2.5.6-fixed-md4/lib/mdfour.h	2003-04-04 14:25:50.000000000 -0800
@@ -21,11 +21,12 @@
 
 struct mdfour {
 	uint32 A, B, C, D;
-	uint32 totalN;
+	uint32 totalN;          /* bit count, lower 32 bits */
+	uint32 totalN2;         /* bit count, upper 32 bits */
 };
 
 void mdfour_begin(struct mdfour *md);
-void mdfour_update(struct mdfour *md, unsigned char *in, int n);
+void mdfour_update(struct mdfour *md, unsigned char *in, uint32 n);
 void mdfour_result(struct mdfour *md, unsigned char *out);
 void mdfour(unsigned char *out, unsigned char *in, int n);