Moving files between dirs / remove-sent

[Wayne Davison]

On Tue, Feb 14, 2006 at 07:17:34PM +0000, rsync at hot.ping.de wrote:
> Wayne, you told me about deleting after the senders get something like an 
> ACK from the receiver, works this really DURING the transfer? 

The detail I got wrong in my last description is that the delay in
deleting files is much worse when no files exist on the receiving side
(not when files exist and are different).  This is because the pipelined
nature of the generator causes it to race ahead requesting files while
the receiving is working on receiving the first file.

One potential way to make this better would be to limit how far ahead of
the receiver the generator is allowed to get.

For instance, the attached patch lets the generator ask for 10 files, at
which point it waits for either a success or redo message before it will
proceed to the next request.  This interleaves the deletes during the
transfer in a much better manner.  The number "10" was not tested to
ensure that it was the best number for the job, but it did work well in
limited testing (i.e. rsync did not slow down appreciably).

..wayne..
-------------- next part --------------
--- generator.c	7 Feb 2006 18:15:46 -0000	1.253
+++ generator.c	14 Feb 2006 21:59:42 -0000
@@ -1231,6 +1231,8 @@ static void recv_generator(char *fname, 
 		rprintf(FINFO, "generating and sending sums for %d\n", ndx);
 
   notify_others:
+	if (remove_sent_files && !phase)
+	    check_completion_progress(itemizing, code);
 	write_int(f_out, ndx);
 	if (itemizing) {
 		int iflags = ITEM_TRANSFER;
--- io.c	4 Feb 2006 21:52:32 -0000	1.179
+++ io.c	14 Feb 2006 21:59:42 -0000
@@ -105,6 +105,7 @@ static char io_filesfrom_lastchar;
 static int io_filesfrom_buflen;
 static size_t contiguous_write_len = 0;
 static int select_timeout = SELECT_TIMEOUT;
+static int pending_files = 0;
 
 static void read_loop(int fd, char *buf, size_t len);
 
@@ -279,6 +280,8 @@ static void read_msg_fd(void)
 			exit_cleanup(RERR_STREAMIO);
 		}
 		read_loop(fd, buf, 4);
+		if (remove_sent_files)
+			pending_files--;
 		flist_ndx_push(&redo_list, IVAL(buf,0));
 		break;
 	case MSG_DELETED:
@@ -295,8 +298,10 @@ static void read_msg_fd(void)
 			exit_cleanup(RERR_STREAMIO);
 		}
 		read_loop(fd, buf, len);
-		if (remove_sent_files)
+		if (remove_sent_files) {
+			pending_files--;
 			io_multiplex_write(MSG_SUCCESS, buf, len);
+		}
 		if (preserve_hard_links)
 			flist_ndx_push(&hlink_list, IVAL(buf,0));
 		break;
@@ -327,6 +332,16 @@ static void read_msg_fd(void)
 	msg_fd_in = fd;
 }
 
+void check_completion_progress(int itemizing, enum logcode code)
+{
+	pending_files++;
+	while (pending_files > 10) {
+		if (hlink_list.head)
+			check_for_finished_hlinks(itemizing, code);
+		read_msg_fd();
+	}
+}
+
 /* Try to push messages off the list onto the wire.  If we leave with more
  * to do, return 0.  On error, return -1.  If everything flushed, return 1.
  * This is only active in the receiver. */
--- proto.h	8 Feb 2006 02:35:25 -0000	1.294
+++ proto.h	14 Feb 2006 21:59:42 -0000
@@ -101,6 +101,7 @@ void io_set_sock_fds(int f_in, int f_out
 void set_io_timeout(int secs);
 void set_msg_fd_in(int fd);
 void set_msg_fd_out(int fd);
+void check_completion_progress(int itemizing, enum logcode code);
 void send_msg(enum msgcode code, char *buf, int len);
 int get_redo_num(int itemizing, enum logcode code);
 int get_hlink_num(void);