[PATCH] Batch-mode rewrite
Chris Shoemaker
c.shoemaker at cox.net
Tue Jul 13 00:03:31 GMT 2004
On Mon, Jul 12, 2004 at 02:47:30PM -0400, Chris Shoemaker wrote:
>
> Ok. "diff -cu" it is. I used -b because the auto-tab feature
> in emacs sometimes causes noisy whitespace changes in the diff.
>
> I'll incorporate your comments and rediff.
>
Ok, actually it seems "diff -cu" isn't right. Instead I just used "diff
-u"
The attached patch attempt to also remove vestigial batch-mode stuff so
it's a bit longer.
Your feedback greatly simplified the checksum-seed and protocol-version
special cases. Thanks for pointing those things out.
Tomorrow, I'll start working on updating the man page.
-chris
-------------- next part --------------
Index: batch.c
===================================================================
RCS file: /cvsroot/rsync/batch.c,v
retrieving revision 1.32
diff -u -r1.32 batch.c
--- batch.c 15 May 2004 19:31:10 -0000 1.32
+++ batch.c 13 Jul 2004 04:41:42 -0000
@@ -13,50 +13,8 @@
extern int protocol_version;
extern struct stats stats;
-struct file_list *batch_flist;
-
-static char rsync_flist_file[] = ".rsync_flist";
-static char rsync_csums_file[] = ".rsync_csums";
-static char rsync_delta_file[] = ".rsync_delta";
static char rsync_argvs_file[] = ".rsync_argvs";
-static int f_csums = -1;
-static int f_delta = -1;
-
-void write_batch_flist_info(int flist_count, struct file_struct **files)
-{
- char filename[MAXPATHLEN];
- int i, f, save_pv;
- int64 save_written;
-
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_flist_file, NULL);
-
- f = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
- if (f < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error", filename);
- exit_cleanup(1);
- }
-
- save_written = stats.total_written;
- save_pv = protocol_version;
- protocol_version = PROTOCOL_VERSION;
- write_int(f, protocol_version);
- write_int(f, flist_count);
-
- for (i = 0; i < flist_count; i++) {
- send_file_entry(files[i], f,
- files[i]->flags & FLAG_TOP_DIR ? XMIT_TOP_DIR : 0);
- }
- send_file_entry(NULL, f, 0);
-
- protocol_version = save_pv;
- stats.total_written = save_written;
-
- close(f);
-}
-
-
void write_batch_argvs_file(int argc, char *argv[])
{
int f;
@@ -116,216 +74,6 @@
close(f);
}
-struct file_list *create_flist_from_batch(void)
-{
- char filename[MAXPATHLEN];
- unsigned short flags;
- int i, f, save_pv;
- int64 save_read;
-
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_flist_file, NULL);
-
- f = do_open(filename, O_RDONLY, 0);
- if (f < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error", filename);
- exit_cleanup(1);
- }
-
- batch_flist = flist_new(WITH_HLINK, "create_flist_from_batch");
-
- save_read = stats.total_read;
- save_pv = protocol_version;
- protocol_version = read_int(f);
-
- batch_flist->count = read_int(f);
- flist_expand(batch_flist);
-
- for (i = 0; (flags = read_byte(f)) != 0; i++) {
- if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
- flags |= read_byte(f) << 8;
- receive_file_entry(&batch_flist->files[i], flags, batch_flist, f);
- }
- receive_file_entry(NULL, 0, NULL, 0); /* Signal that we're done. */
-
- protocol_version = save_pv;
- stats.total_read = save_read;
-
- return batch_flist;
-}
-
-void write_batch_csums_file(void *buff, int bytes_to_write)
-{
- if (write(f_csums, buff, bytes_to_write) < 0) {
- rsyserr(FERROR, errno, "Batch file write error");
- close(f_csums);
- exit_cleanup(1);
- }
-}
-
-void close_batch_csums_file(void)
-{
- close(f_csums);
- f_csums = -1;
-}
-
-
-/**
- * Write csum info to batch file
- *
- * @todo This will break if s->count is ever larger than maxint. The
- * batch code should probably be changed to consistently use the
- * variable-length integer routines, which is probably a compatible
- * change.
- **/
-void write_batch_csum_info(int *flist_entry, struct sum_struct *s)
-{
- size_t i;
- int int_count;
- char filename[MAXPATHLEN];
-
- if (f_csums < 0) {
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_csums_file, NULL);
-
- f_csums = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC,
- S_IRUSR | S_IWUSR);
- if (f_csums < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error",
- filename);
- close(f_csums);
- exit_cleanup(1);
- }
- }
-
- write_batch_csums_file(flist_entry, sizeof (int));
- int_count = s ? (int) s->count : 0;
- write_batch_csums_file(&int_count, sizeof int_count);
-
- if (s) {
- for (i = 0; i < s->count; i++) {
- write_batch_csums_file(&s->sums[i].sum1,
- sizeof (uint32));
- write_batch_csums_file(s->sums[i].sum2, csum_length);
- }
- }
-}
-
-int read_batch_csums_file(char *buff, int len)
-{
- int bytes_read;
-
- if ((bytes_read = read(f_csums, buff, len)) < 0) {
- rsyserr(FERROR, errno, "Batch file read error");
- close(f_csums);
- exit_cleanup(1);
- }
- return bytes_read;
-}
-
-void read_batch_csum_info(int flist_entry, struct sum_struct *s,
- int *checksums_match)
-{
- int i;
- int file_flist_entry;
- int file_chunk_ct;
- uint32 file_sum1;
- char file_sum2[SUM_LENGTH];
- char filename[MAXPATHLEN];
-
- if (f_csums < 0) {
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_csums_file, NULL);
-
- f_csums = do_open(filename, O_RDONLY, 0);
- if (f_csums < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error",
- filename);
- close(f_csums);
- exit_cleanup(1);
- }
- }
-
- read_batch_csums_file((char *) &file_flist_entry, sizeof (int));
- if (file_flist_entry != flist_entry) {
- rprintf(FINFO, "file_flist_entry (%d) != flist_entry (%d)\n",
- file_flist_entry, flist_entry);
- close(f_csums);
- exit_cleanup(1);
-
- } else {
- read_batch_csums_file((char *) &file_chunk_ct, sizeof (int));
- *checksums_match = 1;
- for (i = 0; i < file_chunk_ct; i++) {
- read_batch_csums_file((char *) &file_sum1,
- sizeof (uint32));
- read_batch_csums_file(file_sum2, csum_length);
-
- if ((s->sums[i].sum1 != file_sum1)
- || memcmp(s->sums[i].sum2, file_sum2, csum_length))
- *checksums_match = 0;
- } /* end for */
- }
-}
-
-void write_batch_delta_file(char *buff, int bytes_to_write)
-{
- char filename[MAXPATHLEN];
-
- if (f_delta < 0) {
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_delta_file, NULL);
-
- f_delta = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC,
- S_IRUSR | S_IWUSR);
- if (f_delta < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error",
- filename);
- exit_cleanup(1);
- }
- }
-
- if (write(f_delta, buff, bytes_to_write) < 0) {
- rsyserr(FERROR, errno, "Batch file %s write error", filename);
- close(f_delta);
- exit_cleanup(1);
- }
-}
-
-void close_batch_delta_file(void)
-{
- close(f_delta);
- f_delta = -1;
-}
-
-int read_batch_delta_file(char *buff, int len)
-{
- int bytes_read;
- char filename[MAXPATHLEN];
-
- if (f_delta < 0) {
- stringjoin(filename, sizeof filename,
- batch_prefix, rsync_delta_file, NULL);
-
- f_delta = do_open(filename, O_RDONLY, 0);
- if (f_delta < 0) {
- rsyserr(FERROR, errno, "Batch file %s open error",
- filename);
- close(f_delta);
- exit_cleanup(1);
- }
- }
-
- bytes_read = read(f_delta, buff, len);
- if (bytes_read < 0) {
- rsyserr(FERROR, errno, "Batch file %s read error", filename);
- close(f_delta);
- exit_cleanup(1);
- }
-
- return bytes_read;
-}
-
void show_flist(int index, struct file_struct **fptr)
{
/* for debugging show_flist(flist->count, flist->files * */
Index: clientserver.c
===================================================================
RCS file: /cvsroot/rsync/clientserver.c,v
retrieving revision 1.127
diff -u -r1.127 clientserver.c
--- clientserver.c 13 Jun 2004 14:18:48 -0000 1.127
+++ clientserver.c 13 Jul 2004 04:41:43 -0000
@@ -97,7 +97,8 @@
return ret < 0? ret : client_run(fd, fd, -1, argc, argv);
}
-int start_inband_exchange(char *user, char *path, int f_in, int f_out, int argc)
+int start_inband_exchange(char *user, char *path, int f_in, int f_out,
+ int argc)
{
int i;
char *sargs[MAX_ARGS];
Index: compat.c
===================================================================
RCS file: /cvsroot/rsync/compat.c,v
retrieving revision 1.22
diff -u -r1.22 compat.c
--- compat.c 11 May 2004 17:25:01 -0000 1.22
+++ compat.c 13 Jul 2004 04:41:43 -0000
@@ -28,6 +28,10 @@
int remote_protocol = 0;
extern int am_server;
+extern int am_sender;
+extern int read_batch;
+extern int write_batch;
+extern int batch_fd;
extern int checksum_seed;
@@ -46,6 +50,10 @@
}
if (protocol_version > remote_protocol)
protocol_version = remote_protocol;
+ if ((protocol_version < remote_protocol) & read_batch) {
+ rprintf(FERROR, "Batch file contains protocol version %d, but this version of rsync only supports up to protocol version %d. Please upgrade rsync.\n", remote_protocol, protocol_version);
+ exit_cleanup(RERR_PROTOCOL);
+ }
}
if (verbose > 3) {
@@ -73,6 +81,39 @@
exit_cleanup(RERR_PROTOCOL);
}
+ /* CAS: I think this is a good candidate for a protocol
+ * change. Instead of:
+ *
+ * if (am_server) write_int() stuff;
+ * else read_int() stuff;
+ *
+ * it is good to remove the protocol dependence on server-ness:
+ *
+ * if (am_sender) write_int() stuff;
+ * else read_int() stuff;
+ *
+ * This has the effect of halving the number of possible
+ * protocol combinations, because sender and receiver already
+ * have an asymetric protocol, and varying on server-ness
+ * means 2 more possibilities for each side.
+ *
+ * Another reason: the priviledge of choosing the checksum
+ * seed should be given to the party that may have a greater
+ * interest in specifying the actual seed value. The only
+ * reason to be interested in the selection of seed value is
+ * if you have cached checksums. However, the sender has the
+ * greater interest because a file sent can be sent again with
+ * the same checksum values, but a file received has no valid
+ * cached checksums. (Specifically, if the file was modified
+ * then the checksums cached become invalid, and if the file
+ * was new then no checksums are cached.)
+ *
+ * One could argue that the receiver could want to cache
+ * checksums for the case where a receiver-server is
+ * frequently checksumming a file but infrequently updating
+ * it. Surely this is obscure.
+ */
+
if (am_server) {
if (!checksum_seed)
checksum_seed = time(NULL);
@@ -80,4 +121,17 @@
} else {
checksum_seed = read_int(f_in);
}
+
+ /* Some communication has already taken place, but we don't
+ * enable batch writing until here so that we can write a
+ * canonical record of the communication even though the
+ * actual communication so far depends on whether a server is
+ * involved.
+ */
+ if (write_batch) {
+ write_int(batch_fd, protocol_version);
+ write_int(batch_fd, checksum_seed);
+ write_batch_start(f_out, f_in);
+ }
}
+
Index: flist.c
===================================================================
RCS file: /cvsroot/rsync/flist.c,v
retrieving revision 1.231
diff -u -r1.231 flist.c
--- flist.c 18 Jun 2004 16:29:21 -0000 1.231
+++ flist.c 13 Jul 2004 04:41:44 -0000
@@ -64,9 +64,6 @@
extern int orig_umask;
extern int list_only;
-extern int read_batch;
-extern int write_batch;
-
extern struct exclude_list_struct exclude_list;
extern struct exclude_list_struct server_exclude_list;
extern struct exclude_list_struct local_exclude_list;
@@ -950,9 +947,6 @@
flist_expand(flist);
- if (write_batch)
- file->flags |= FLAG_TOP_DIR;
-
if (file->basename[0]) {
flist->files[flist->count++] = file;
send_file_entry(file, f, base_flags);
@@ -1231,8 +1225,6 @@
io_end_buffering();
stats.flist_size = stats.total_written - start_write;
stats.num_files = flist->count;
- if (write_batch)
- write_batch_flist_info(flist->count, flist->files);
}
if (verbose > 3)
@@ -1297,17 +1289,16 @@
clean_flist(flist, relative_paths, 1);
if (f != -1) {
- /* Now send the uid/gid list. This was introduced in
+ /* Now send the uid/gid list. This was introduced in
* protocol version 15 */
recv_uid_list(f, flist);
- if (!read_batch) {
- /* Recv the io_error flag */
- if (lp_ignore_errors(module_id) || ignore_errors)
- read_int(f);
- else
- io_error |= read_int(f);
- }
+ /* Recv the io_error flag */
+ if (lp_ignore_errors(module_id) || ignore_errors)
+ read_int(f);
+ else
+ io_error |= read_int(f);
+
}
if (verbose > 3)
Index: io.c
===================================================================
RCS file: /cvsroot/rsync/io.c,v
retrieving revision 1.131
diff -u -r1.131 io.c
--- io.c 23 Jun 2004 01:13:09 -0000 1.131
+++ io.c 13 Jul 2004 04:41:45 -0000
@@ -57,6 +57,12 @@
extern char *remote_filesfrom_file;
extern struct stats stats;
+extern int batch_fd;
+extern int write_batch;
+static int write_batch_monitor_in = -1;
+static int write_batch_monitor_out = -1;
+
+
const char phase_unknown[] = "unknown";
int select_timeout = SELECT_TIMEOUT;
@@ -674,6 +680,13 @@
total += ret;
}
+ if (fd == write_batch_monitor_in) {
+ if (write(batch_fd, buffer, total) < 0) {
+ close(batch_fd);
+ exit_cleanup(RERR_FILEIO);
+ }
+ }
+
stats.total_read += total;
}
@@ -951,6 +964,13 @@
exit_cleanup(RERR_PROTOCOL);
}
+ if (fd == write_batch_monitor_out) {
+ if (write(batch_fd, buf, len) < 0) {
+ close(batch_fd);
+ exit_cleanup(RERR_FILEIO);
+ }
+ }
+
if (!io_buffer || fd != multiplex_out_fd) {
writefd_unbuffered(fd, buf, len);
return;
@@ -1109,3 +1129,16 @@
io_multiplexing_out = 0;
}
+void write_batch_start(int f_out, int f_in)
+{
+ if (am_sender)
+ write_batch_monitor_out = f_out;
+ else
+ write_batch_monitor_in = f_in;
+}
+
+void write_batch_stop(void)
+{
+ write_batch_monitor_out = -1;
+ write_batch_monitor_in = -1;
+}
Index: main.c
===================================================================
RCS file: /cvsroot/rsync/main.c,v
retrieving revision 1.203
diff -u -r1.203 main.c
--- main.c 13 Jul 2004 01:44:03 -0000 1.203
+++ main.c 13 Jul 2004 04:41:46 -0000
@@ -58,7 +58,8 @@
extern char *remote_filesfrom_file;
extern char *rsync_path;
extern char *shell_cmd;
-extern struct file_list *batch_flist;
+extern int batch_fd;
+extern char *batch_prefix;
/* there's probably never more than at most 2 outstanding child processes,
@@ -126,6 +127,25 @@
return;
}
+ /* CAS: I think that this is a good candidate for a protocol
+ * change. Instead of making the protocol depend on
+ * am_server, I think this should be a simple:
+ *
+ * if (am_sender)
+ * write_longint stuff;
+ * else
+ * read_longing stuff;
+ *
+ * This would have several advantages: 1) It simplifies the
+ * protocol by removing variation based on server-ness. 2) It
+ * makes the client_run()-if(am_sender) path look more like
+ * do_server_sender (ideally, these should be the same). 3)
+ * It removes the need for write_batch corrections that have
+ * to record a canonical (independent of server-ness) record
+ * of the transmission, and read_batch corrections that have
+ * to replay this canonical record.
+ */
+
if (am_server) {
if (am_sender) {
int64 w;
@@ -301,8 +321,6 @@
}
if (local_server) {
- if (read_batch)
- create_flist_from_batch(); /* sets batch_flist */
/* If the user didn't request --[no-]whole-file, force
* it on, but only if we're not batch processing. */
if (whole_file < 0 && !read_batch && !write_batch)
@@ -466,7 +484,13 @@
recv_files(f_in,flist,local_name);
io_flush(FULL_FLUSH);
- report(f_in);
+
+ /* This test for read_batch is needed because of a
+ * protocol dependency on am_server state, see
+ * report(). We are quite fortunate that this
+ * workaround is not more complicated. */
+ if (!read_batch)
+ report(f_in);
send_msg(MSG_DONE, "", 0);
io_flush(FULL_FLUSH);
@@ -478,6 +502,7 @@
}
am_generator = 1;
+ write_batch_stop();
close(error_pipe[1]);
if (f_in != f_out)
@@ -548,10 +573,7 @@
filesfrom_fd = -1;
}
- if (read_batch)
- flist = batch_flist;
- else
- flist = recv_file_list(f_in);
+ flist = recv_file_list(f_in);
if (!flist) {
rprintf(FERROR,"server_recv: recv_file_list error\n");
exit_cleanup(RERR_FILESELECT);
@@ -590,11 +612,10 @@
if (am_sender) {
keep_dirlinks = 0; /* Must be disabled on the sender. */
- if (!read_batch) {
- recv_exclude_list(f_in);
- if (cvs_exclude)
- add_cvs_excludes();
- }
+
+ recv_exclude_list(f_in);
+ if (cvs_exclude)
+ add_cvs_excludes();
do_server_sender(f_in, f_out, argc, argv);
} else {
do_server_recv(f_in, f_out, argc, argv);
@@ -614,15 +635,23 @@
char *local_name = NULL;
cleanup_child_pid = pid;
- if (read_batch)
- flist = batch_flist;
-
- set_nonblocking(f_in);
- set_nonblocking(f_out);
+ if (read_batch) {
+ /* This is the heart of the read_batch approach:
+ * Switcher-roo the file descriptors, and
+ * nobody's the wiser. */
+ close(f_in);
+ close(f_out);
+ f_in = batch_fd;
+ f_out = do_open("/dev/null", O_WRONLY, 0);
+ am_sender = 0;
+ } else {
+ set_nonblocking(f_in);
+ set_nonblocking(f_out);
+ }
- setup_protocol(f_out,f_in);
+ setup_protocol(f_out, f_in);
- if (protocol_version >= 23)
+ if (protocol_version >= 23 && !read_batch)
io_start_multiplex_in(f_in);
if (am_sender) {
@@ -634,6 +663,9 @@
send_exclude_list(f_out);
if (remote_filesfrom_file)
filesfrom_fd = f_in;
+
+ /* Can be unconditional, but this is theoretically
+ * more efficent for read_batch case. */
if (!read_batch) /* don't write to pipe */
flist = send_file_list(f_out,argc,argv);
if (verbose > 3)
@@ -642,6 +674,8 @@
io_flush(NORMAL_FLUSH);
send_files(flist,f_out,f_in);
io_flush(FULL_FLUSH);
+ /* Wondering why this doesn't look like do_server_sender()?
+ * see report(). */
if (protocol_version >= 24) {
/* final goodbye message */
read_int(f_in);
@@ -660,6 +694,8 @@
if (argc == 0)
list_only = 1;
+ /* Can be unconditional, but this is theoretically more
+ * efficient for the read_batch case. */
if (!read_batch)
send_exclude_list(f_out);
@@ -731,7 +767,8 @@
return rc;
/* rsync:// always uses rsync server over direct socket connection */
- if (strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0) {
+ if (strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0 &&
+ !read_batch) {
char *host, *path;
host = argv[0] + strlen(URL_PREFIX);
@@ -844,9 +881,15 @@
}
argc--;
} else { /* read_batch */
- am_sender = 1;
local_server = 1;
shell_path = argv[argc-1];
+ /* need to complain if dest is remote */
+ p = find_colon(argv[0]);
+ if ((strncasecmp(URL_PREFIX, argv[0], strlen(URL_PREFIX)) == 0)
+ || p) {
+ rprintf(FERROR, "remote destination is not allowed with --read-batch\n");
+ exit_cleanup(RERR_SYNTAX);
+ }
}
if (shell_machine) {
@@ -1042,8 +1085,18 @@
init_flist();
- if (write_batch && !am_server) {
- write_batch_argvs_file(orig_argc, orig_argv);
+ if (write_batch || read_batch) {
+ if (write_batch)
+ write_batch_argvs_file(orig_argc, orig_argv);
+
+ batch_fd = do_open(batch_prefix,
+ write_batch ? O_WRONLY | O_CREAT | O_TRUNC
+ : O_RDONLY, S_IRUSR | S_IWUSR);
+ if (batch_fd < 0) {
+ rsyserr(FERROR, errno, "Batch file %s open error",
+ batch_prefix);
+ exit_cleanup(RERR_FILEIO);
+ }
}
if (am_daemon && !am_server)
Index: match.c
===================================================================
RCS file: /cvsroot/rsync/match.c,v
retrieving revision 1.65
diff -u -r1.65 match.c
--- match.c 7 Jul 2004 08:38:40 -0000 1.65
+++ match.c 13 Jul 2004 04:41:46 -0000
@@ -285,7 +285,6 @@
void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
{
char file_sum[MD4_SUM_LENGTH];
- extern int write_batch;
last_match = 0;
false_alarms = 0;
@@ -323,8 +322,6 @@
if (verbose > 2)
rprintf(FINFO,"sending file_sum\n");
write_buf(f,file_sum,MD4_SUM_LENGTH);
- if (write_batch)
- write_batch_delta_file(file_sum, MD4_SUM_LENGTH);
if (targets) {
free(targets);
Index: options.c
===================================================================
RCS file: /cvsroot/rsync/options.c,v
retrieving revision 1.157
diff -u -r1.157 options.c
--- options.c 20 Jun 2004 19:47:05 -0000 1.157
+++ options.c 13 Jul 2004 04:41:47 -0000
@@ -111,6 +111,7 @@
int write_batch = 0;
int read_batch = 0;
+int batch_fd = 0;
int backup_dir_len = 0;
int backup_suffix_len;
unsigned int backup_dir_remainder;
@@ -638,6 +639,16 @@
}
#endif
+ if ((write_batch || read_batch) && am_server) {
+ rprintf(FERROR,
+ "batch-mode is incompatible with server mode\n");
+ write_batch = 0;
+ read_batch = 0;
+ /* We don't actually exit_cleanup(), so that we can still service
+ * older version clients that still send batch args to server */
+
+ }
+
if (write_batch && read_batch) {
rprintf(FERROR,
"write-batch and read-batch can not be used together\n");
@@ -655,12 +666,6 @@
exit_cleanup(RERR_SYNTAX);
}
- if (do_compression && (write_batch || read_batch)) {
- rprintf(FERROR,
- "compress can not be used with write-batch or read-batch\n");
- exit_cleanup(RERR_SYNTAX);
- }
-
if (archive_mode) {
if (!files_from)
recurse = 1;
@@ -884,12 +889,7 @@
args[ac++] = arg;
}
- if (batch_prefix) {
- char *r_or_w = write_batch ? "write" : "read";
- if (asprintf(&arg, "--%s-batch=%s", r_or_w, batch_prefix) < 0)
- goto oom;
- args[ac++] = arg;
- }
+ /* batch operations are local only, so don't pass batch args */
if (io_timeout) {
if (asprintf(&arg, "--timeout=%d", io_timeout) < 0)
Index: pipe.c
===================================================================
RCS file: /cvsroot/rsync/pipe.c,v
retrieving revision 1.8
diff -u -r1.8 pipe.c
--- pipe.c 18 Jun 2004 16:00:33 -0000 1.8
+++ pipe.c 13 Jul 2004 04:41:47 -0000
@@ -26,6 +26,7 @@
extern int blocking_io;
extern int orig_umask;
extern int read_batch;
+extern int write_batch;
extern int filesfrom_fd;
/**
@@ -94,7 +95,19 @@
return pid;
}
-pid_t local_child(int argc, char **argv,int *f_in,int *f_out,
+/*
+ * This function forks a child which calls child_main(). First,
+ * however, it has to establish communication paths to and from the
+ * newborn child. It creates two socket pairs -- one for writing to
+ * the child (from the parent) and one for reading from the child
+ * (writing to the parent). Since that's four socket ends, each
+ * process has to close the two ends it doesn't need. The remaining
+ * two socket ends are retained for reading and writing. In the
+ * child, the STDIN and STDOUT file descriptors refer to these
+ * sockets. In the parent, the function arguments f_in and f_out are
+ * set to refer to these sockets.
+ */
+pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
int (*child_main)(int, char*[]))
{
pid_t pid;
@@ -107,7 +120,9 @@
exit_cleanup(RERR_IPC);
}
- pid = do_fork();
+ /* For read-batch, don't even fork. */
+ pid = read_batch ? getpid() : do_fork();
+
if (pid == -1) {
rsyserr(FERROR, errno, "fork");
exit_cleanup(RERR_IPC);
@@ -132,6 +147,14 @@
child_main(argc, argv);
}
+ /* Either sender or receiver can write the file for
+ * write_batch mode. In local_child, both are local
+ * processes, so we must make sure that only one
+ * actually writes. It shouldn't matter which one,
+ * because they must produce the same file.
+ */
+ write_batch = 0;
+
if (!am_sender)
filesfrom_fd = -1;
Index: proto.h
===================================================================
RCS file: /cvsroot/rsync/proto.h,v
retrieving revision 1.198
diff -u -r1.198 proto.h
--- proto.h 29 Jun 2004 19:18:55 -0000 1.198
+++ proto.h 13 Jul 2004 04:41:48 -0000
@@ -5,18 +5,9 @@
char *auth_server(int f_in, int f_out, int module, char *addr, char *leader);
void auth_client(int fd, char *user, char *challenge);
int make_backup(char *fname);
-void write_batch_flist_info(int flist_count, struct file_struct **files);
void write_batch_argvs_file(int argc, char *argv[]);
-struct file_list *create_flist_from_batch(void);
-void write_batch_csums_file(void *buff, int bytes_to_write);
-void close_batch_csums_file(void);
-void write_batch_csum_info(int *flist_entry, struct sum_struct *s);
-int read_batch_csums_file(char *buff, int len);
-void read_batch_csum_info(int flist_entry, struct sum_struct *s,
- int *checksums_match);
-void write_batch_delta_file(char *buff, int bytes_to_write);
-void close_batch_delta_file(void);
-int read_batch_delta_file(char *buff, int len);
+void write_batch_start(int f_out, int f_in);
+void write_batch_stop(void);
void show_flist(int index, struct file_struct **fptr);
void show_argvs(int argc, char *argv[]);
uint32 get_checksum1(char *buf1,int len);
Index: receiver.c
===================================================================
RCS file: /cvsroot/rsync/receiver.c,v
retrieving revision 1.86
diff -u -r1.86 receiver.c
--- receiver.c 2 Jul 2004 18:23:57 -0000 1.86
+++ receiver.c 13 Jul 2004 04:41:48 -0000
@@ -289,7 +289,7 @@
* main routine for receiver process.
*
* Receiver process runs on the same host as the generator process. */
-int recv_files(int f_in,struct file_list *flist,char *local_name)
+int recv_files(int f_in, struct file_list *flist, char *local_name)
{
int fd1,fd2;
STRUCT_STAT st;
Index: sender.c
===================================================================
RCS file: /cvsroot/rsync/sender.c,v
retrieving revision 1.41
diff -u -r1.41 sender.c
--- sender.c 18 Jun 2004 16:55:12 -0000 1.41
+++ sender.c 13 Jul 2004 04:41:48 -0000
@@ -118,13 +118,7 @@
int phase = 0;
extern struct stats stats;
struct stats initial_stats;
- extern int write_batch;
- extern int read_batch;
- int checksums_match;
- int buff_len;
- char buff[CHUNK_SIZE];
int j;
- int done;
if (verbose > 2)
rprintf(FINFO, "send_files starting\n");
@@ -185,117 +179,69 @@
rprintf(FERROR, "receive_sums failed\n");
return;
}
-
- if (write_batch)
- write_batch_csum_info(&i, s);
-
- if (!read_batch) {
- fd = do_open(fname, O_RDONLY, 0);
- if (fd == -1) {
- if (errno == ENOENT) {
- enum logcode c = am_daemon
- && protocol_version < 28 ? FERROR
- : FINFO;
- io_error |= IOERR_VANISHED;
- rprintf(c, "file has vanished: %s\n",
- full_fname(fname));
- } else {
- io_error |= IOERR_GENERAL;
- rsyserr(FERROR, errno,
- "send_files failed to open %s",
- full_fname(fname));
- }
- free_sums(s);
- continue;
- }
-
- /* map the local file */
- if (do_fstat(fd, &st) != 0) {
+
+ fd = do_open(fname, O_RDONLY, 0);
+ if (fd == -1) {
+ if (errno == ENOENT) {
+ enum logcode c = am_daemon
+ && protocol_version < 28 ? FERROR
+ : FINFO;
+ io_error |= IOERR_VANISHED;
+ rprintf(c, "file has vanished: %s\n",
+ full_fname(fname));
+ } else {
io_error |= IOERR_GENERAL;
- rsyserr(FERROR, errno, "fstat failed");
- free_sums(s);
- close(fd);
- return;
- }
-
- mbuf = st.st_size ? map_file(fd, st.st_size) : NULL;
-
- if (verbose > 2) {
- rprintf(FINFO, "send_files mapped %s of size %.0f\n",
- fname, (double)st.st_size);
+ rsyserr(FERROR, errno,
+ "send_files failed to open %s",
+ full_fname(fname));
}
-
- write_int(f_out, i);
-
- if (write_batch)
- write_batch_delta_file((char *)&i, sizeof i);
-
- write_sum_head(f_out, s);
+ free_sums(s);
+ continue;
}
-
- if (verbose > 2 && !read_batch)
+
+ /* map the local file */
+ if (do_fstat(fd, &st) != 0) {
+ io_error |= IOERR_GENERAL;
+ rsyserr(FERROR, errno, "fstat failed");
+ free_sums(s);
+ close(fd);
+ return;
+ }
+
+ mbuf = st.st_size ? map_file(fd, st.st_size) : NULL;
+
+ if (verbose > 2) {
+ rprintf(FINFO, "send_files mapped %s of size %.0f\n",
+ fname, (double)st.st_size);
+ }
+
+ write_int(f_out, i);
+
+ write_sum_head(f_out, s);
+
+ if (verbose > 2)
rprintf(FINFO, "calling match_sums %s\n", fname);
-
+
if (!am_server && verbose) {
rprintf(FINFO, "%s\n", fname+offset);
}
set_compression(fname);
- if (read_batch) {
- /* read checksums originally computed on sender side */
- read_batch_csum_info(i, s, &checksums_match);
- if (checksums_match) {
- read_batch_delta_file((char*)&j, sizeof (int));
- if (j != i) { /* if flist index entries don't match*/
- rprintf(FINFO, "index mismatch in send_files\n");
- rprintf(FINFO, "read index = %d flist ndx = %d\n", j, i);
- close_batch_delta_file();
- close_batch_csums_file();
- exit_cleanup(1);
- } else {
- write_int(f_out, j);
- write_sum_head(f_out, s);
- done = 0;
- while (!done) {
- read_batch_delta_file((char*)&buff_len, sizeof (int));
- write_int(f_out, buff_len);
- if (buff_len == 0) {
- done = 1;
- } else {
- if (buff_len > 0) {
- read_batch_delta_file(buff, buff_len);
- write_buf(f_out, buff, buff_len);
- }
- }
- }
- read_batch_delta_file(buff, MD4_SUM_LENGTH);
- write_buf(f_out, buff, MD4_SUM_LENGTH);
-
- }
- } else { /* not checksum match */
- rprintf (FINFO, "readbatch & checksums don't match\n");
- rprintf (FINFO, "filename=%s is being skipped\n", fname);
- continue;
- }
- } else { /* not read_batch */
- match_sums(f_out, s, mbuf, st.st_size);
- log_send(file, &initial_stats);
- }
+ match_sums(f_out, s, mbuf, st.st_size);
+ log_send(file, &initial_stats);
- if (!read_batch) {
- if (mbuf) {
- j = unmap_file(mbuf);
- if (j) {
- io_error |= IOERR_GENERAL;
- rsyserr(FERROR, j,
- "read errors mapping %s",
- full_fname(fname));
- }
+ if (mbuf) {
+ j = unmap_file(mbuf);
+ if (j) {
+ io_error |= IOERR_GENERAL;
+ rsyserr(FERROR, j,
+ "read errors mapping %s",
+ full_fname(fname));
}
- close(fd);
}
-
+ close(fd);
+
free_sums(s);
if (verbose > 2)
@@ -308,10 +254,6 @@
match_report();
write_int(f_out, -1);
- if (write_batch || read_batch) {
- close_batch_csums_file();
- close_batch_delta_file();
- }
}
Index: token.c
===================================================================
RCS file: /cvsroot/rsync/token.c,v
retrieving revision 1.31
diff -u -r1.31 token.c
--- token.c 18 Jun 2004 16:22:14 -0000 1.31
+++ token.c 13 Jul 2004 04:41:49 -0000
@@ -22,7 +22,6 @@
extern int do_compression;
extern int module_id;
-extern int write_batch;
static int compression_level = Z_DEFAULT_COMPRESSION;
@@ -105,20 +104,12 @@
int n1 = MIN(CHUNK_SIZE,n-l);
write_int(f,n1);
write_buf(f,map_ptr(buf,offset+l,n1),n1);
- if (write_batch) {
- write_batch_delta_file( (char *) &n1, sizeof(int) );
- write_batch_delta_file(map_ptr(buf,offset+l,n1),n1);
- }
l += n1;
}
}
/* a -2 token means to send data only and no token */
if (token != -2) {
write_int(f,-(token+1));
- if (write_batch) {
- hold_int = -(token+1);
- write_batch_delta_file( (char *) &hold_int, sizeof(int) );
- }
}
}
@@ -198,28 +189,13 @@
n = last_token - run_start;
if (r >= 0 && r <= 63) {
write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);
- if (write_batch) {
- temp_byte = (char)( (n==0? TOKEN_REL: TOKENRUN_REL) + r);
- write_batch_delta_file(&temp_byte,sizeof(char));
- }
} else {
write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));
write_int(f, run_start);
- if (write_batch) {
- temp_byte = (char)(n==0? TOKEN_LONG: TOKENRUN_LONG);
- write_batch_delta_file(&temp_byte,sizeof(char));
- write_batch_delta_file((char *)&run_start,sizeof(run_start));
- }
}
if (n != 0) {
write_byte(f, n);
write_byte(f, n >> 8);
- if (write_batch) {
- temp_byte = (char)n;
- write_batch_delta_file(&temp_byte,sizeof(char));
- temp_byte = (char)(n >> 8);
- write_batch_delta_file(&temp_byte,sizeof(char));
- }
}
last_run_end = last_token;
run_start = token;
@@ -278,8 +254,6 @@
obuf[0] = DEFLATED_DATA + (n >> 8);
obuf[1] = n;
write_buf(f, obuf, n+2);
- if (write_batch)
- write_batch_delta_file(obuf,n+2);
}
}
} while (nb != 0 || tx_strm.avail_out == 0);
@@ -289,11 +263,6 @@
if (token == -1) {
/* end of file - clean up */
write_byte(f, END_FLAG);
- if (write_batch) {
- temp_byte = END_FLAG;
- write_batch_delta_file(&temp_byte,sizeof(char));
- }
-
} else if (token != -2) {
/* add the data in the current block to the compressor's
history and hash table */
More information about the rsync
mailing list