[PATCH] Batch-mode rewrite

Chris Shoemaker c.shoemaker at cox.net
Tue Jul 13 00:03:31 GMT 2004


On Mon, Jul 12, 2004 at 02:47:30PM -0400, Chris Shoemaker wrote:
> 
> Ok.   "diff -cu" it is.  I used -b because the auto-tab feature
> in emacs sometimes causes noisy whitespace changes in the diff.
> 
> I'll incorporate your comments and rediff.
>

Ok, actually it seems "diff -cu" isn't right.  Instead I just used "diff
-u"

The attached patch attempt to also remove vestigial batch-mode stuff so
it's a bit longer.

Your feedback greatly simplified the checksum-seed and protocol-version
special cases.  Thanks for pointing those things out.

Tomorrow, I'll start working on updating the man page.

-chris

-------------- next part --------------
Index: batch.c
===================================================================
RCS file: /cvsroot/rsync/batch.c,v
retrieving revision 1.32
diff -u -r1.32 batch.c
--- batch.c	15 May 2004 19:31:10 -0000	1.32
+++ batch.c	13 Jul 2004 04:41:42 -0000
@@ -13,50 +13,8 @@
 extern int protocol_version;
 extern struct stats stats;
 
-struct file_list *batch_flist;
-
-static char rsync_flist_file[] = ".rsync_flist";
-static char rsync_csums_file[] = ".rsync_csums";
-static char rsync_delta_file[] = ".rsync_delta";
 static char rsync_argvs_file[] = ".rsync_argvs";
 
-static int f_csums = -1;
-static int f_delta = -1;
-
-void write_batch_flist_info(int flist_count, struct file_struct **files)
-{
-	char filename[MAXPATHLEN];
-	int i, f, save_pv;
-	int64 save_written;
-
-	stringjoin(filename, sizeof filename,
-	    batch_prefix, rsync_flist_file, NULL);
-
-	f = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
-	if (f < 0) {
-		rsyserr(FERROR, errno, "Batch file %s open error", filename);
-		exit_cleanup(1);
-	}
-
-	save_written = stats.total_written;
-	save_pv = protocol_version;
-	protocol_version = PROTOCOL_VERSION;
-	write_int(f, protocol_version);
-	write_int(f, flist_count);
-
-	for (i = 0; i < flist_count; i++) {
-		send_file_entry(files[i], f,
-		    files[i]->flags & FLAG_TOP_DIR ?  XMIT_TOP_DIR : 0);
-	}
-	send_file_entry(NULL, f, 0);
-
-	protocol_version = save_pv;
-	stats.total_written = save_written;
-
-	close(f);
-}
-
-
 void write_batch_argvs_file(int argc, char *argv[])
 {
 	int f;
@@ -116,216 +74,6 @@
 	close(f);
 }
 
-struct file_list *create_flist_from_batch(void)
-{
-	char filename[MAXPATHLEN];
-	unsigned short flags;
-	int i, f, save_pv;
-	int64 save_read;
-
-	stringjoin(filename, sizeof filename,
-	    batch_prefix, rsync_flist_file, NULL);
-
-	f = do_open(filename, O_RDONLY, 0);
-	if (f < 0) {
-		rsyserr(FERROR, errno, "Batch file %s open error", filename);
-		exit_cleanup(1);
-	}
-
-	batch_flist = flist_new(WITH_HLINK, "create_flist_from_batch");
-
-	save_read = stats.total_read;
-	save_pv = protocol_version;
-	protocol_version = read_int(f);
-
-	batch_flist->count = read_int(f);
-	flist_expand(batch_flist);
-
-	for (i = 0; (flags = read_byte(f)) != 0; i++) {
-		if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
-			flags |= read_byte(f) << 8;
-		receive_file_entry(&batch_flist->files[i], flags, batch_flist, f);
-	}
-	receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */
-
-	protocol_version = save_pv;
-	stats.total_read = save_read;
-
-	return batch_flist;
-}
-
-void write_batch_csums_file(void *buff, int bytes_to_write)
-{
-	if (write(f_csums, buff, bytes_to_write) < 0) {
-		rsyserr(FERROR, errno, "Batch file write error");
-		close(f_csums);
-		exit_cleanup(1);
-	}
-}
-
-void close_batch_csums_file(void)
-{
-	close(f_csums);
-	f_csums = -1;
-}
-
-
-/**
- * Write csum info to batch file
- *
- * @todo This will break if s->count is ever larger than maxint.  The
- * batch code should probably be changed to consistently use the
- * variable-length integer routines, which is probably a compatible
- * change.
- **/
-void write_batch_csum_info(int *flist_entry, struct sum_struct *s)
-{
-	size_t i;
-	int int_count;
-	char filename[MAXPATHLEN];
-
-	if (f_csums < 0) {
-		stringjoin(filename, sizeof filename,
-		    batch_prefix, rsync_csums_file, NULL);
-
-		f_csums = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC,
-		    S_IRUSR | S_IWUSR);
-		if (f_csums < 0) {
-			rsyserr(FERROR, errno, "Batch file %s open error",
-				filename);
-			close(f_csums);
-			exit_cleanup(1);
-		}
-	}
-
-	write_batch_csums_file(flist_entry, sizeof (int));
-	int_count = s ? (int) s->count : 0;
-	write_batch_csums_file(&int_count, sizeof int_count);
-
-	if (s) {
-		for (i = 0; i < s->count; i++) {
-			write_batch_csums_file(&s->sums[i].sum1,
-			    sizeof (uint32));
-			write_batch_csums_file(s->sums[i].sum2, csum_length);
-		}
-	}
-}
-
-int read_batch_csums_file(char *buff, int len)
-{
-	int bytes_read;
-
-	if ((bytes_read = read(f_csums, buff, len)) < 0) {
-		rsyserr(FERROR, errno, "Batch file read error");
-		close(f_csums);
-		exit_cleanup(1);
-	}
-	return bytes_read;
-}
-
-void read_batch_csum_info(int flist_entry, struct sum_struct *s,
-			  int *checksums_match)
-{
-	int i;
-	int file_flist_entry;
-	int file_chunk_ct;
-	uint32 file_sum1;
-	char file_sum2[SUM_LENGTH];
-	char filename[MAXPATHLEN];
-
-	if (f_csums < 0) {
-		stringjoin(filename, sizeof filename,
-		    batch_prefix, rsync_csums_file, NULL);
-
-		f_csums = do_open(filename, O_RDONLY, 0);
-		if (f_csums < 0) {
-			rsyserr(FERROR, errno, "Batch file %s open error",
-				filename);
-			close(f_csums);
-			exit_cleanup(1);
-		}
-	}
-
-	read_batch_csums_file((char *) &file_flist_entry, sizeof (int));
-	if (file_flist_entry != flist_entry) {
-		rprintf(FINFO, "file_flist_entry (%d) != flist_entry (%d)\n",
-		    file_flist_entry, flist_entry);
-		close(f_csums);
-		exit_cleanup(1);
-
-	} else {
-		read_batch_csums_file((char *) &file_chunk_ct, sizeof (int));
-		*checksums_match = 1;
-		for (i = 0; i < file_chunk_ct; i++) {
-			read_batch_csums_file((char *) &file_sum1,
-			    sizeof (uint32));
-			read_batch_csums_file(file_sum2, csum_length);
-
-			if ((s->sums[i].sum1 != file_sum1)
-			    || memcmp(s->sums[i].sum2, file_sum2, csum_length))
-				*checksums_match = 0;
-		}		/*  end for  */
-	}
-}
-
-void write_batch_delta_file(char *buff, int bytes_to_write)
-{
-	char filename[MAXPATHLEN];
-
-	if (f_delta < 0) {
-		stringjoin(filename, sizeof filename,
-		    batch_prefix, rsync_delta_file, NULL);
-
-		f_delta = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC,
-				  S_IRUSR | S_IWUSR);
-		if (f_delta < 0) {
-			rsyserr(FERROR, errno, "Batch file %s open error",
-				filename);
-			exit_cleanup(1);
-		}
-	}
-
-	if (write(f_delta, buff, bytes_to_write) < 0) {
-		rsyserr(FERROR, errno, "Batch file %s write error", filename);
-		close(f_delta);
-		exit_cleanup(1);
-	}
-}
-
-void close_batch_delta_file(void)
-{
-	close(f_delta);
-	f_delta = -1;
-}
-
-int read_batch_delta_file(char *buff, int len)
-{
-	int bytes_read;
-	char filename[MAXPATHLEN];
-
-	if (f_delta < 0) {
-		stringjoin(filename, sizeof filename,
-		    batch_prefix, rsync_delta_file, NULL);
-
-		f_delta = do_open(filename, O_RDONLY, 0);
-		if (f_delta < 0) {
-			rsyserr(FERROR, errno, "Batch file %s open error",
-				filename);
-			close(f_delta);
-			exit_cleanup(1);
-		}
-	}
-
-	bytes_read = read(f_delta, buff, len);
-	if (bytes_read < 0) {
-		rsyserr(FERROR, errno, "Batch file %s read error", filename);
-		close(f_delta);
-		exit_cleanup(1);
-	}
-
-	return bytes_read;
-}
-
 void show_flist(int index, struct file_struct **fptr)
 {
 	/*  for debugging    show_flist(flist->count, flist->files * */
Index: clientserver.c
===================================================================
RCS file: /cvsroot/rsync/clientserver.c,v
retrieving revision 1.127
diff -u -r1.127 clientserver.c
--- clientserver.c	13 Jun 2004 14:18:48 -0000	1.127
+++ clientserver.c	13 Jul 2004 04:41:43 -0000
@@ -97,7 +97,8 @@
 	return ret < 0? ret : client_run(fd, fd, -1, argc, argv);
 }
 
-int start_inband_exchange(char *user, char *path, int f_in, int f_out, int argc)
+int start_inband_exchange(char *user, char *path, int f_in, int f_out, 
+			  int argc)
 {
 	int i;
 	char *sargs[MAX_ARGS];
Index: compat.c
===================================================================
RCS file: /cvsroot/rsync/compat.c,v
retrieving revision 1.22
diff -u -r1.22 compat.c
--- compat.c	11 May 2004 17:25:01 -0000	1.22
+++ compat.c	13 Jul 2004 04:41:43 -0000
@@ -28,6 +28,10 @@
 int remote_protocol = 0;
 
 extern int am_server;
+extern int am_sender;
+extern int read_batch;
+extern int write_batch;
+extern int batch_fd;
 
 extern int checksum_seed;
 
@@ -46,6 +50,10 @@
 		}
 		if (protocol_version > remote_protocol)
 			protocol_version = remote_protocol;
+		if ((protocol_version < remote_protocol) & read_batch) {
+		        rprintf(FERROR, "Batch file contains protocol version %d, but this version of rsync only supports up to protocol version %d.  Please upgrade rsync.\n", remote_protocol, protocol_version);
+			exit_cleanup(RERR_PROTOCOL);
+		}
 	}
 
 	if (verbose > 3) {
@@ -73,6 +81,39 @@
 		exit_cleanup(RERR_PROTOCOL);
 	}
 
+	/* CAS: I think this is a good candidate for a protocol
+	 * change.  Instead of:
+	 *
+	 *   if (am_server) write_int() stuff;
+	 *   else read_int() stuff;
+	 *
+	 * it is good to remove the protocol dependence on server-ness:
+	 * 
+	 *   if (am_sender) write_int() stuff;
+	 *   else read_int() stuff;
+	 *
+	 * This has the effect of halving the number of possible
+	 * protocol combinations, because sender and receiver already
+	 * have an asymetric protocol, and varying on server-ness
+	 * means 2 more possibilities for each side.
+	 *
+	 * Another reason: the priviledge of choosing the checksum
+	 * seed should be given to the party that may have a greater
+	 * interest in specifying the actual seed value.  The only
+	 * reason to be interested in the selection of seed value is
+	 * if you have cached checksums.  However, the sender has the
+	 * greater interest because a file sent can be sent again with
+	 * the same checksum values, but a file received has no valid
+	 * cached checksums.  (Specifically, if the file was modified
+	 * then the checksums cached become invalid, and if the file
+	 * was new then no checksums are cached.)
+	 *
+	 * One could argue that the receiver could want to cache
+	 * checksums for the case where a receiver-server is
+	 * frequently checksumming a file but infrequently updating
+	 * it.  Surely this is obscure.
+	 */
+
 	if (am_server) {
 		if (!checksum_seed)
 			checksum_seed = time(NULL);
@@ -80,4 +121,17 @@
 	} else {
 		checksum_seed = read_int(f_in);
 	}
+
+	/* Some communication has already taken place, but we don't
+	 * enable batch writing until here so that we can write a
+	 * canonical record of the communication even though the
+	 * actual communication so far depends on whether a server is
+	 * involved.
+	 */
+	if (write_batch) { 
+		write_int(batch_fd, protocol_version);
+		write_int(batch_fd, checksum_seed);
+		write_batch_start(f_out, f_in);
+	}
 }
+
Index: flist.c
===================================================================
RCS file: /cvsroot/rsync/flist.c,v
retrieving revision 1.231
diff -u -r1.231 flist.c
--- flist.c	18 Jun 2004 16:29:21 -0000	1.231
+++ flist.c	13 Jul 2004 04:41:44 -0000
@@ -64,9 +64,6 @@
 extern int orig_umask;
 extern int list_only;
 
-extern int read_batch;
-extern int write_batch;
-
 extern struct exclude_list_struct exclude_list;
 extern struct exclude_list_struct server_exclude_list;
 extern struct exclude_list_struct local_exclude_list;
@@ -950,9 +947,6 @@
 
 	flist_expand(flist);
 
-	if (write_batch)
-		file->flags |= FLAG_TOP_DIR;
-
 	if (file->basename[0]) {
 		flist->files[flist->count++] = file;
 		send_file_entry(file, f, base_flags);
@@ -1231,8 +1225,6 @@
 		io_end_buffering();
 		stats.flist_size = stats.total_written - start_write;
 		stats.num_files = flist->count;
-		if (write_batch)
-			write_batch_flist_info(flist->count, flist->files);
 	}
 
 	if (verbose > 3)
@@ -1297,17 +1289,16 @@
 	clean_flist(flist, relative_paths, 1);
 
 	if (f != -1) {
-		/* Now send the uid/gid list. This was introduced in
+	        /* Now send the uid/gid list. This was introduced in 
 		 * protocol version 15 */
 		recv_uid_list(f, flist);
 
-		if (!read_batch) {
-			/* Recv the io_error flag */
-			if (lp_ignore_errors(module_id) || ignore_errors)
-				read_int(f);
-			else
-				io_error |= read_int(f);
-		}
+		/* Recv the io_error flag */
+		if (lp_ignore_errors(module_id) || ignore_errors)
+		  read_int(f);
+		else
+		  io_error |= read_int(f);
+		
 	}
 
 	if (verbose > 3)
Index: io.c
===================================================================
RCS file: /cvsroot/rsync/io.c,v
retrieving revision 1.131
diff -u -r1.131 io.c
--- io.c	23 Jun 2004 01:13:09 -0000	1.131
+++ io.c	13 Jul 2004 04:41:45 -0000
@@ -57,6 +57,12 @@
 extern char *remote_filesfrom_file;
 extern struct stats stats;
 
+extern int batch_fd;
+extern int write_batch;
+static int write_batch_monitor_in = -1;
+static int write_batch_monitor_out = -1;
+
+
 const char phase_unknown[] = "unknown";
 int select_timeout = SELECT_TIMEOUT;
 
@@ -674,6 +680,13 @@
 		total += ret;
 	}
 
+	if (fd == write_batch_monitor_in) {
+		if (write(batch_fd, buffer, total) < 0) {
+			close(batch_fd);
+			exit_cleanup(RERR_FILEIO);
+		}
+	}
+	
 	stats.total_read += total;
 }
 
@@ -951,6 +964,13 @@
 		exit_cleanup(RERR_PROTOCOL);
 	}
 
+	if (fd == write_batch_monitor_out) {
+		if (write(batch_fd, buf, len) < 0) {
+			close(batch_fd);
+			exit_cleanup(RERR_FILEIO);
+		}
+	}
+
 	if (!io_buffer || fd != multiplex_out_fd) {
 		writefd_unbuffered(fd, buf, len);
 		return;
@@ -1109,3 +1129,16 @@
 	io_multiplexing_out = 0;
 }
 
+void write_batch_start(int f_out, int f_in)
+{
+	if (am_sender)
+		write_batch_monitor_out = f_out;
+	else
+		write_batch_monitor_in = f_in;
+}
+
+void write_batch_stop(void)
+{
+	write_batch_monitor_out = -1;
+	write_batch_monitor_in = -1;
+}
Index: main.c
===================================================================
RCS file: /cvsroot/rsync/main.c,v
retrieving revision 1.203
diff -u -r1.203 main.c
--- main.c	13 Jul 2004 01:44:03 -0000	1.203
+++ main.c	13 Jul 2004 04:41:46 -0000
@@ -58,7 +58,8 @@
 extern char *remote_filesfrom_file;
 extern char *rsync_path;
 extern char *shell_cmd;
-extern struct file_list *batch_flist;
+extern int batch_fd;
+extern char *batch_prefix;
 
 
 /* there's probably never more than at most 2 outstanding child processes,
@@ -126,6 +127,25 @@
 			return;
 	}
 
+	/* CAS: I think that this is a good candidate for a protocol
+	 * change.  Instead of making the protocol depend on
+	 * am_server, I think this should be a simple:
+	 *
+	 *  if (am_sender)
+	 *     write_longint stuff;
+	 *  else
+	 *     read_longing stuff;
+	 *
+	 * This would have several advantages: 1) It simplifies the
+	 * protocol by removing variation based on server-ness.  2) It
+	 * makes the client_run()-if(am_sender) path look more like
+	 * do_server_sender (ideally, these should be the same).  3)
+	 * It removes the need for write_batch corrections that have
+	 * to record a canonical (independent of server-ness) record
+	 * of the transmission, and read_batch corrections that have
+	 * to replay this canonical record.
+	 */
+
 	if (am_server) {
 		if (am_sender) {
 			int64 w;
@@ -301,8 +321,6 @@
 	}
 
 	if (local_server) {
-		if (read_batch)
-			create_flist_from_batch(); /* sets batch_flist */
 		/* If the user didn't request --[no-]whole-file, force
 		 * it on, but only if we're not batch processing. */
 		if (whole_file < 0 && !read_batch && !write_batch)
@@ -466,7 +484,13 @@
 
 		recv_files(f_in,flist,local_name);
 		io_flush(FULL_FLUSH);
-		report(f_in);
+
+		/* This test for read_batch is needed because of a
+		 * protocol dependency on am_server state, see
+		 * report().  We are quite fortunate that this
+		 * workaround is not more complicated. */
+		if (!read_batch)
+			report(f_in);
 
 		send_msg(MSG_DONE, "", 0);
 		io_flush(FULL_FLUSH);
@@ -478,6 +502,7 @@
 	}
 
 	am_generator = 1;
+	write_batch_stop();
 
 	close(error_pipe[1]);
 	if (f_in != f_out)
@@ -548,10 +573,7 @@
 		filesfrom_fd = -1;
 	}
 
-	if (read_batch)
-		flist = batch_flist;
-	else
-		flist = recv_file_list(f_in);
+	flist = recv_file_list(f_in);
 	if (!flist) {
 		rprintf(FERROR,"server_recv: recv_file_list error\n");
 		exit_cleanup(RERR_FILESELECT);
@@ -590,11 +612,10 @@
 
 	if (am_sender) {
 		keep_dirlinks = 0; /* Must be disabled on the sender. */
-		if (!read_batch) {
-			recv_exclude_list(f_in);
-			if (cvs_exclude)
-				add_cvs_excludes();
-		}
+		
+		recv_exclude_list(f_in);
+		if (cvs_exclude)
+			add_cvs_excludes();
 		do_server_sender(f_in, f_out, argc, argv);
 	} else {
 		do_server_recv(f_in, f_out, argc, argv);
@@ -614,15 +635,23 @@
 	char *local_name = NULL;
 
 	cleanup_child_pid = pid;
-	if (read_batch)
-		flist = batch_flist;
-
-	set_nonblocking(f_in);
-	set_nonblocking(f_out);
+	if (read_batch) {
+		/* This is the heart of the read_batch approach:
+		 * Switcher-roo the file descriptors, and 
+		 * nobody's the wiser. */
+		close(f_in);
+		close(f_out);
+		f_in = batch_fd;
+		f_out = do_open("/dev/null", O_WRONLY, 0);
+		am_sender = 0;
+	} else {
+		set_nonblocking(f_in);
+		set_nonblocking(f_out);
+	}
 
-	setup_protocol(f_out,f_in);
+	setup_protocol(f_out, f_in);
 
-	if (protocol_version >= 23)
+	if (protocol_version >= 23 && !read_batch)
 		io_start_multiplex_in(f_in);
 
 	if (am_sender) {
@@ -634,6 +663,9 @@
 			send_exclude_list(f_out);
 		if (remote_filesfrom_file)
 			filesfrom_fd = f_in;
+
+		/* Can be unconditional, but this is theoretically
+		 * more efficent for read_batch case. */
 		if (!read_batch) /* don't write to pipe */
 			flist = send_file_list(f_out,argc,argv);
 		if (verbose > 3)
@@ -642,6 +674,8 @@
 		io_flush(NORMAL_FLUSH);
 		send_files(flist,f_out,f_in);
 		io_flush(FULL_FLUSH);
+		/* Wondering why this doesn't look like do_server_sender()?
+		 * see report(). */
 		if (protocol_version >= 24) {
 			/* final goodbye message */
 			read_int(f_in);
@@ -660,6 +694,8 @@
 	if (argc == 0)
 		list_only = 1;
 
+	/* Can be unconditional, but this is theoretically more
+	 * efficient for the read_batch case. */
 	if (!read_batch)
 		send_exclude_list(f_out);
 
@@ -731,7 +767,8 @@
 		return rc;
 
 	/* rsync:// always uses rsync server over direct socket connection */
-	if (strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0) {
+	if (strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0 && 
+	    !read_batch) {
 		char *host, *path;
 
 		host = argv[0] + strlen(URL_PREFIX);
@@ -844,9 +881,15 @@
 		}
 		argc--;
 	} else {  /* read_batch */
-		am_sender = 1;
 		local_server = 1;
 		shell_path = argv[argc-1];
+		/* need to complain if dest is remote */
+		p = find_colon(argv[0]);
+		if ((strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0)
+		    || p) {
+			rprintf(FERROR, "remote destination is not allowed with --read-batch\n");
+			exit_cleanup(RERR_SYNTAX);
+		}
 	}
 
 	if (shell_machine) {
@@ -1042,8 +1085,18 @@
 
 	init_flist();
 
-	if (write_batch && !am_server) {
-		write_batch_argvs_file(orig_argc, orig_argv);
+	if (write_batch || read_batch) {
+		if (write_batch)
+			write_batch_argvs_file(orig_argc, orig_argv);
+
+		batch_fd = do_open(batch_prefix, 
+				   write_batch ? O_WRONLY | O_CREAT | O_TRUNC 
+				   : O_RDONLY, S_IRUSR | S_IWUSR);
+		if (batch_fd < 0) {
+			rsyserr(FERROR, errno, "Batch file %s open error",
+				batch_prefix);
+			exit_cleanup(RERR_FILEIO);
+		}
 	}
 
 	if (am_daemon && !am_server)
Index: match.c
===================================================================
RCS file: /cvsroot/rsync/match.c,v
retrieving revision 1.65
diff -u -r1.65 match.c
--- match.c	7 Jul 2004 08:38:40 -0000	1.65
+++ match.c	13 Jul 2004 04:41:46 -0000
@@ -285,7 +285,6 @@
 void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
 {
 	char file_sum[MD4_SUM_LENGTH];
-	extern int write_batch;
 
 	last_match = 0;
 	false_alarms = 0;
@@ -323,8 +322,6 @@
 	if (verbose > 2)
 		rprintf(FINFO,"sending file_sum\n");
 	write_buf(f,file_sum,MD4_SUM_LENGTH);
-	if (write_batch)
-		write_batch_delta_file(file_sum, MD4_SUM_LENGTH);
 
 	if (targets) {
 		free(targets);
Index: options.c
===================================================================
RCS file: /cvsroot/rsync/options.c,v
retrieving revision 1.157
diff -u -r1.157 options.c
--- options.c	20 Jun 2004 19:47:05 -0000	1.157
+++ options.c	13 Jul 2004 04:41:47 -0000
@@ -111,6 +111,7 @@
 
 int write_batch = 0;
 int read_batch = 0;
+int batch_fd = 0;
 int backup_dir_len = 0;
 int backup_suffix_len;
 unsigned int backup_dir_remainder;
@@ -638,6 +639,16 @@
 	}
 #endif
 
+	if ((write_batch || read_batch) && am_server) {
+		rprintf(FERROR,
+			"batch-mode is incompatible with server mode\n");
+		write_batch = 0;
+		read_batch = 0;
+		/* We don't actually exit_cleanup(), so that we can still service
+		 * older version clients that still send batch args to server */
+		
+	}
+
 	if (write_batch && read_batch) {
 		rprintf(FERROR,
 			"write-batch and read-batch can not be used together\n");
@@ -655,12 +666,6 @@
 		exit_cleanup(RERR_SYNTAX);
 	}
 
-	if (do_compression && (write_batch || read_batch)) {
-		rprintf(FERROR,
-			"compress can not be used with write-batch or read-batch\n");
-		exit_cleanup(RERR_SYNTAX);
-	}
-
 	if (archive_mode) {
 		if (!files_from)
 			recurse = 1;
@@ -884,12 +889,7 @@
 		args[ac++] = arg;
 	}
 
-	if (batch_prefix) {
-		char *r_or_w = write_batch ? "write" : "read";
-		if (asprintf(&arg, "--%s-batch=%s", r_or_w, batch_prefix) < 0)
-			goto oom;
-		args[ac++] = arg;
-	}
+        /* batch operations are local only, so don't pass batch args */
 
 	if (io_timeout) {
 		if (asprintf(&arg, "--timeout=%d", io_timeout) < 0)
Index: pipe.c
===================================================================
RCS file: /cvsroot/rsync/pipe.c,v
retrieving revision 1.8
diff -u -r1.8 pipe.c
--- pipe.c	18 Jun 2004 16:00:33 -0000	1.8
+++ pipe.c	13 Jul 2004 04:41:47 -0000
@@ -26,6 +26,7 @@
 extern int blocking_io;
 extern int orig_umask;
 extern int read_batch;
+extern int write_batch;
 extern int filesfrom_fd;
 
 /**
@@ -94,7 +95,19 @@
 	return pid;
 }
 
-pid_t local_child(int argc, char **argv,int *f_in,int *f_out,
+/*
+ * This function forks a child which calls child_main().  First,
+ * however, it has to establish communication paths to and from the
+ * newborn child.  It creates two socket pairs -- one for writing to
+ * the child (from the parent) and one for reading from the child
+ * (writing to the parent).  Since that's four socket ends, each
+ * process has to close the two ends it doesn't need.  The remaining
+ * two socket ends are retained for reading and writing.  In the
+ * child, the STDIN and STDOUT file descriptors refer to these
+ * sockets.  In the parent, the function arguments f_in and f_out are
+ * set to refer to these sockets.
+ */
+pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
 		  int (*child_main)(int, char*[]))
 {
 	pid_t pid;
@@ -107,7 +120,9 @@
 		exit_cleanup(RERR_IPC);
 	}
 
-	pid = do_fork();
+	/* For read-batch, don't even fork. */
+	pid = read_batch ? getpid() : do_fork();
+
 	if (pid == -1) {
 		rsyserr(FERROR, errno, "fork");
 		exit_cleanup(RERR_IPC);
@@ -132,6 +147,14 @@
 		child_main(argc, argv);
 	}
 
+	/* Either sender or receiver can write the file for
+	 * write_batch mode.  In local_child, both are local
+	 * processes, so we must make sure that only one
+	 * actually writes.  It shouldn't matter which one,
+	 * because they must produce the same file.
+	 */
+	write_batch = 0;
+
 	if (!am_sender)
 		filesfrom_fd = -1;
 
Index: proto.h
===================================================================
RCS file: /cvsroot/rsync/proto.h,v
retrieving revision 1.198
diff -u -r1.198 proto.h
--- proto.h	29 Jun 2004 19:18:55 -0000	1.198
+++ proto.h	13 Jul 2004 04:41:48 -0000
@@ -5,18 +5,9 @@
 char *auth_server(int f_in, int f_out, int module, char *addr, char *leader);
 void auth_client(int fd, char *user, char *challenge);
 int make_backup(char *fname);
-void write_batch_flist_info(int flist_count, struct file_struct **files);
 void write_batch_argvs_file(int argc, char *argv[]);
-struct file_list *create_flist_from_batch(void);
-void write_batch_csums_file(void *buff, int bytes_to_write);
-void close_batch_csums_file(void);
-void write_batch_csum_info(int *flist_entry, struct sum_struct *s);
-int read_batch_csums_file(char *buff, int len);
-void read_batch_csum_info(int flist_entry, struct sum_struct *s,
-			  int *checksums_match);
-void write_batch_delta_file(char *buff, int bytes_to_write);
-void close_batch_delta_file(void);
-int read_batch_delta_file(char *buff, int len);
+void write_batch_start(int f_out, int f_in);
+void write_batch_stop(void);
 void show_flist(int index, struct file_struct **fptr);
 void show_argvs(int argc, char *argv[]);
 uint32 get_checksum1(char *buf1,int len);
Index: receiver.c
===================================================================
RCS file: /cvsroot/rsync/receiver.c,v
retrieving revision 1.86
diff -u -r1.86 receiver.c
--- receiver.c	2 Jul 2004 18:23:57 -0000	1.86
+++ receiver.c	13 Jul 2004 04:41:48 -0000
@@ -289,7 +289,7 @@
  * main routine for receiver process.
  *
  * Receiver process runs on the same host as the generator process. */
-int recv_files(int f_in,struct file_list *flist,char *local_name)
+int recv_files(int f_in, struct file_list *flist, char *local_name)
 {
 	int fd1,fd2;
 	STRUCT_STAT st;
Index: sender.c
===================================================================
RCS file: /cvsroot/rsync/sender.c,v
retrieving revision 1.41
diff -u -r1.41 sender.c
--- sender.c	18 Jun 2004 16:55:12 -0000	1.41
+++ sender.c	13 Jul 2004 04:41:48 -0000
@@ -118,13 +118,7 @@
 	int phase = 0;
 	extern struct stats stats;
 	struct stats initial_stats;
-	extern int write_batch;
-	extern int read_batch;
-	int checksums_match;
-	int buff_len;
-	char buff[CHUNK_SIZE];
 	int j;
-	int done;
 
 	if (verbose > 2)
 		rprintf(FINFO, "send_files starting\n");
@@ -185,117 +179,69 @@
 			rprintf(FERROR, "receive_sums failed\n");
 			return;
 		}
-
-		if (write_batch)
-			write_batch_csum_info(&i, s);
-
-		if (!read_batch) {
-			fd = do_open(fname, O_RDONLY, 0);
-			if (fd == -1) {
-				if (errno == ENOENT) {
-					enum logcode c = am_daemon
-					    && protocol_version < 28 ? FERROR
-								     : FINFO;
-					io_error |= IOERR_VANISHED;
-					rprintf(c, "file has vanished: %s\n",
-						full_fname(fname));
-				} else {
-					io_error |= IOERR_GENERAL;
-					rsyserr(FERROR, errno,
-						"send_files failed to open %s",
-						full_fname(fname));
-				}
-				free_sums(s);
-				continue;
-			}
-
-			/* map the local file */
-			if (do_fstat(fd, &st) != 0) {
+		
+		fd = do_open(fname, O_RDONLY, 0);
+		if (fd == -1) {
+			if (errno == ENOENT) {
+				enum logcode c = am_daemon
+					&& protocol_version < 28 ? FERROR
+					: FINFO;
+				io_error |= IOERR_VANISHED;
+				rprintf(c, "file has vanished: %s\n",
+					full_fname(fname));
+			} else {
 				io_error |= IOERR_GENERAL;
-				rsyserr(FERROR, errno, "fstat failed");
-				free_sums(s);
-				close(fd);
-				return;
-			}
-
-			mbuf = st.st_size ? map_file(fd, st.st_size) : NULL;
-
-			if (verbose > 2) {
-				rprintf(FINFO, "send_files mapped %s of size %.0f\n",
-					fname, (double)st.st_size);
+				rsyserr(FERROR, errno,
+					"send_files failed to open %s",
+					full_fname(fname));
 			}
-
-			write_int(f_out, i);
-
-			if (write_batch)
-				write_batch_delta_file((char *)&i, sizeof i);
-
-			write_sum_head(f_out, s);
+			free_sums(s);
+			continue;
 		}
-
-		if (verbose > 2 && !read_batch)
+		
+		/* map the local file */
+		if (do_fstat(fd, &st) != 0) {
+			io_error |= IOERR_GENERAL;
+			rsyserr(FERROR, errno, "fstat failed");
+			free_sums(s);
+			close(fd);
+			return;
+		}
+		
+		mbuf = st.st_size ? map_file(fd, st.st_size) : NULL;
+		
+		if (verbose > 2) {
+			rprintf(FINFO, "send_files mapped %s of size %.0f\n",
+				fname, (double)st.st_size);
+		}
+		
+		write_int(f_out, i);
+		
+		write_sum_head(f_out, s);
+		
+		if (verbose > 2)
 			rprintf(FINFO, "calling match_sums %s\n", fname);
-
+		
 		if (!am_server && verbose) {
 			rprintf(FINFO, "%s\n", fname+offset);
 		}
 
 		set_compression(fname);
 
-		if (read_batch) {
-			/* read checksums originally computed on sender side */
-			read_batch_csum_info(i, s, &checksums_match);
-			if (checksums_match) {
-				read_batch_delta_file((char*)&j, sizeof (int));
-				if (j != i) {    /* if flist index entries don't match*/
-					rprintf(FINFO, "index mismatch in send_files\n");
-					rprintf(FINFO, "read index = %d flist ndx = %d\n", j, i);
-					close_batch_delta_file();
-					close_batch_csums_file();
-					exit_cleanup(1);
-				} else {
-					write_int(f_out, j);
-					write_sum_head(f_out, s);
-					done = 0;
-					while (!done) {
-						read_batch_delta_file((char*)&buff_len, sizeof (int));
-						write_int(f_out, buff_len);
-						if (buff_len == 0) {
-							done = 1;
-						} else {
-							if (buff_len > 0) {
-								read_batch_delta_file(buff, buff_len);
-								write_buf(f_out, buff, buff_len);
-							}
-						}
-					}
-					read_batch_delta_file(buff, MD4_SUM_LENGTH);
-					write_buf(f_out, buff, MD4_SUM_LENGTH);
-
-				}
-			} else {  /* not checksum match */
-				rprintf (FINFO, "readbatch & checksums don't match\n");
-				rprintf (FINFO, "filename=%s is being skipped\n", fname);
-				continue;
-			}
-		} else  { /* not read_batch */
-			match_sums(f_out, s, mbuf, st.st_size);
-			log_send(file, &initial_stats);
-		}
+		match_sums(f_out, s, mbuf, st.st_size);
+		log_send(file, &initial_stats);
 
-		if (!read_batch) {
-			if (mbuf) {
-				j = unmap_file(mbuf);
-				if (j) {
-					io_error |= IOERR_GENERAL;
-					rsyserr(FERROR, j,
-						"read errors mapping %s",
-						full_fname(fname));
-				}
+		if (mbuf) {
+			j = unmap_file(mbuf);
+			if (j) {
+				io_error |= IOERR_GENERAL;
+				rsyserr(FERROR, j,
+					"read errors mapping %s",
+					full_fname(fname));
 			}
-			close(fd);
 		}
-
+		close(fd);
+		
 		free_sums(s);
 
 		if (verbose > 2)
@@ -308,10 +254,6 @@
 	match_report();
 
 	write_int(f_out, -1);
-	if (write_batch || read_batch) {
-		close_batch_csums_file();
-		close_batch_delta_file();
-	}
 
 }
 
Index: token.c
===================================================================
RCS file: /cvsroot/rsync/token.c,v
retrieving revision 1.31
diff -u -r1.31 token.c
--- token.c	18 Jun 2004 16:22:14 -0000	1.31
+++ token.c	13 Jul 2004 04:41:49 -0000
@@ -22,7 +22,6 @@
 
 extern int do_compression;
 extern int module_id;
-extern int write_batch;
 
 static int compression_level = Z_DEFAULT_COMPRESSION;
 
@@ -105,20 +104,12 @@
 			int n1 = MIN(CHUNK_SIZE,n-l);
 			write_int(f,n1);
 			write_buf(f,map_ptr(buf,offset+l,n1),n1);
-			if (write_batch) {
-				write_batch_delta_file( (char *) &n1, sizeof(int) );
-				write_batch_delta_file(map_ptr(buf,offset+l,n1),n1);
-			}
 			l += n1;
 		}
 	}
 	/* a -2 token means to send data only and no token */
 	if (token != -2) {
 		write_int(f,-(token+1));
-		if (write_batch) {
-			hold_int = -(token+1);
-			write_batch_delta_file( (char *) &hold_int, sizeof(int) );
-		}
 	}
 }
 
@@ -198,28 +189,13 @@
 		n = last_token - run_start;
 		if (r >= 0 && r <= 63) {
 			write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
-			if (write_batch) {
-				temp_byte = (char)( (n==0? TOKEN_REL: TOKENRUN_REL) + r);
-				write_batch_delta_file(&temp_byte,sizeof(char));
-			}
 		} else {
 			write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
 			write_int(f, run_start);
-			if (write_batch) {
-				temp_byte = (char)(n==0? TOKEN_LONG: TOKENRUN_LONG);
-				write_batch_delta_file(&temp_byte,sizeof(char));
-				write_batch_delta_file((char *)&run_start,sizeof(run_start));
-			}
 		}
 		if (n != 0) {
 			write_byte(f, n);
 			write_byte(f, n >> 8);
-			if (write_batch) {
-				temp_byte = (char)n;
-				write_batch_delta_file(&temp_byte,sizeof(char));
-				temp_byte = (char)(n >> 8);
-				write_batch_delta_file(&temp_byte,sizeof(char));
-			}
 		}
 		last_run_end = last_token;
 		run_start = token;
@@ -278,8 +254,6 @@
 					obuf[0] = DEFLATED_DATA + (n >> 8);
 					obuf[1] = n;
 					write_buf(f, obuf, n+2);
-					if (write_batch)
-						write_batch_delta_file(obuf,n+2);
 				}
 			}
 		} while (nb != 0 || tx_strm.avail_out == 0);
@@ -289,11 +263,6 @@
 	if (token == -1) {
 		/* end of file - clean up */
 		write_byte(f, END_FLAG);
-		if (write_batch) {
-			temp_byte = END_FLAG;
-			write_batch_delta_file(&temp_byte,sizeof(char));
-		}
-
 	} else if (token != -2) {
 		/* add the data in the current block to the compressor's
 		   history and hash table */


More information about the rsync mailing list