--delete-sent-files (AKA --move-files)
Wayne Davison
wayned at samba.org
Sat Jan 17 05:30:57 GMT 2004
Yes, it's time once again to return to the subject of moving files.
With the recent changes to the communications code between the receiver
and the generator, there is now a non-clogging channel that we can use
to signal the sender when a file has been successfully transferred,
which allows us delete the original for all transferred files. I have
in the past waffled on whether this feature needs to be in rsync. I'm
currently back on the side that it's a nice thing to support. YMMV.
Here's a new implementation of the feature that adds a generic message
(MSG_SUCCESS) for the receiver to send back to the sender (through the
generator). I made the generator simply forward this message on to the
sender, but to do that it means that the generator must be in multi-
plexed IO mode, which it used to be only when it was on the server side.
My patch adds a new internal flag that lets the code request that the
generator have messaging capability even when it is on the client side
(the non-delete-specific variable makes the code more generic). The one
not-so-nice thing about this setup is that the sender process gets the
MSG_SUCCESS for a particular index when it is down in the I/O code, and
that code doesn't know about the file list. I decided to make this code
call a new function, successful_send(), which is back in sender.c. This
function is the one that handles translating the index into a file name
and deleting the source file (assuming that the delete_sent_files flag
is on, which is currently the only time that MSG_SUCCESS gets sent). I
also added a run-time flag to mark the items we sent off to the
receiver, just to make sure that nothing funny is going on in the
sequence of events (aside: the sender side has no copy-on-write issues
to make us avoid tweaking the flags).
So, feel free to take a look and see if you like what I've done.
..wayne..
-------------- next part --------------
Index: flist.c
--- flist.c 17 Jan 2004 01:16:49 -0000 1.165
+++ flist.c 17 Jan 2004 05:04:54 -0000
@@ -602,7 +602,7 @@ void receive_file_entry(struct file_stru
if (!file->basename)
out_of_memory("receive_file_entry 1");
- file->flags = flags;
+ file->flags = flags & LIVE_FLAGS;
file->length = read_longint(f);
if (!(flags & SAME_TIME))
modtime = (time_t)read_int(f);
Index: io.c
--- io.c 16 Jan 2004 16:31:47 -0000 1.119
+++ io.c 17 Jan 2004 05:04:54 -0000
@@ -222,6 +222,14 @@ static void read_msg_fd(void)
read_loop(fd, buf, 4);
redo_list_add(IVAL(buf,0));
break;
+ case MSG_SUCCESS:
+ if (len != 4) {
+ rprintf(FERROR, "invalid message %d:%d\n", tag, len);
+ exit_cleanup(RERR_STREAMIO);
+ }
+ read_loop(fd, buf, 4);
+ io_multiplex_write(MSG_SUCCESS, buf, 4);
+ break;
case MSG_INFO:
case MSG_ERROR:
case MSG_LOG:
@@ -637,6 +645,16 @@ static int read_unbuffered(int fd, char
}
read_loop(fd, buffer, remaining);
bufferIdx = 0;
+ break;
+ case MSG_SUCCESS:
+ if (remaining != 4) {
+ rprintf(FERROR, "invalid multi-message %d:%ld\n",
+ tag, (long)remaining);
+ exit_cleanup(RERR_STREAMIO);
+ }
+ read_loop(fd, line, 4);
+ successful_send(IVAL(line, 0));
+ remaining = 0;
break;
case MSG_INFO:
case MSG_ERROR:
Index: main.c
--- main.c 17 Jan 2004 05:04:04 -0000 1.181
+++ main.c 17 Jan 2004 05:04:54 -0000
@@ -41,6 +41,7 @@ extern int list_only;
extern int local_server;
extern int log_got_error;
extern int module_id;
+extern int need_messages_from_generator;
extern int orig_umask;
extern int preserve_hard_links;
extern int protocol_version;
@@ -558,6 +559,8 @@ void start_server(int f_in, int f_out, i
io_start_multiplex_out(f_out);
if (am_sender) {
+ if (need_messages_from_generator)
+ io_start_multiplex_in(f_in);
if (!read_batch) {
recv_exclude_list(f_in);
if (cvs_exclude)
@@ -623,6 +626,9 @@ int client_run(int f_in, int f_out, pid_
io_flush(FULL_FLUSH);
exit_cleanup(status);
}
+
+ if (need_messages_from_generator)
+ io_start_multiplex_out(f_out);
if (argc == 0) {
list_only = 1;
Index: options.c
--- options.c 15 Jan 2004 17:43:34 -0000 1.124
+++ options.c 17 Jan 2004 05:04:55 -0000
@@ -81,12 +81,14 @@ int copy_unsafe_links=0;
int size_only=0;
int bwlimit=0;
int delete_after=0;
+int delete_sent_files = 0;
int only_existing=0;
int opt_ignore_existing=0;
int max_delete=0;
int ignore_errors=0;
int modify_window=0;
int blocking_io=-1;
+int need_messages_from_generator = 0;
unsigned int block_size = 0;
@@ -245,6 +247,7 @@ void usage(enum logcode F)
rprintf(F," --delete delete files that don't exist on the sending side\n");
rprintf(F," --delete-excluded also delete excluded files on the receiving side\n");
rprintf(F," --delete-after receiver deletes after transferring, not before\n");
+ rprintf(F," --delete-sent-files updated/sent files are removed from sending side\n");
rprintf(F," --ignore-errors delete even if there are IO errors\n");
rprintf(F," --max-delete=NUM don't delete more than NUM files\n");
rprintf(F," --partial keep partially transferred files\n");
@@ -294,8 +297,8 @@ void usage(enum logcode F)
}
enum {OPT_VERSION = 1000, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
- OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_LINK_DEST,
- OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
+ OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_DELETE_SENT_FILES,
+ OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_LINK_DEST, OPT_MODIFY_WINDOW,
OPT_READ_BATCH, OPT_WRITE_BATCH};
static struct poptOption long_options[] = {
@@ -313,6 +316,7 @@ static struct poptOption long_options[]
{"ignore-existing", 0, POPT_ARG_NONE, &opt_ignore_existing, 0, 0, 0 },
{"delete-after", 0, POPT_ARG_NONE, 0, OPT_DELETE_AFTER, 0, 0 },
{"delete-excluded", 0, POPT_ARG_NONE, 0, OPT_DELETE_EXCLUDED, 0, 0 },
+ {"delete-sent-files",0, POPT_ARG_NONE, 0, OPT_DELETE_SENT_FILES, 0, 0 },
{"force", 0, POPT_ARG_NONE, &force_delete, 0, 0, 0 },
{"numeric-ids", 0, POPT_ARG_NONE, &numeric_ids, 0, 0, 0 },
{"exclude", 0, POPT_ARG_STRING, 0, OPT_EXCLUDE, 0, 0 },
@@ -498,6 +502,11 @@ int parse_arguments(int *argc, const cha
delete_mode = 1;
break;
+ case OPT_DELETE_SENT_FILES:
+ delete_sent_files = 1;
+ need_messages_from_generator = 1;
+ break;
+
case OPT_EXCLUDE:
add_exclude(&exclude_list, poptGetOptArg(pc),
ADD_EXCLUDE);
@@ -902,6 +911,9 @@ void server_options(char **args,int *arg
args[ac++] = "--from0";
}
}
+
+ if (delete_sent_files)
+ args[ac++] = "--delete-sent-files";
*argc = ac;
}
Index: proto.h
--- proto.h 15 Jan 2004 07:42:23 -0000 1.172
+++ proto.h 17 Jan 2004 05:04:55 -0000
@@ -193,6 +193,7 @@ int set_perms(char *fname,struct file_st
void sig_int(void);
void finish_transfer(char *fname, char *fnametmp, struct file_struct *file);
void read_sum_head(int f, struct sum_struct *sum);
+void successful_send(int i);
void send_files(struct file_list *flist, int f_out, int f_in);
int try_bind_local(int s, int ai_family, int ai_socktype,
const char *bind_address);
Index: receiver.c
--- receiver.c 15 Jan 2004 07:42:25 -0000 1.63
+++ receiver.c 17 Jan 2004 05:04:55 -0000
@@ -39,6 +39,7 @@ extern char *backup_dir;
extern char *backup_suffix;
extern int backup_suffix_len;
extern int cleanup_got_literal;
+extern int delete_sent_files;
static void delete_one(char *fn, int is_dir)
{
@@ -287,7 +288,7 @@ int recv_files(int f_in,struct file_list
char *fname, fbuf[MAXPATHLEN];
char template[MAXPATHLEN];
char fnametmp[MAXPATHLEN];
- char *fnamecmp;
+ char *fnamecmp, numbuf[4];
char fnamecmpbuf[MAXPATHLEN];
struct map_struct *mapbuf;
int i;
@@ -461,16 +462,20 @@ int recv_files(int f_in,struct file_list
cleanup_disable();
- if (!recv_ok) {
+ if (recv_ok) {
+ if (delete_sent_files) {
+ SIVAL(numbuf, 0, i);
+ send_msg(MSG_SUCCESS, numbuf, 4);
+ }
+ } else {
if (csum_length == SUM_LENGTH) {
rprintf(FERROR,"ERROR: file corruption in %s. File changed during transfer?\n",
full_fname(fname));
} else {
- char buf[4];
if (verbose > 1)
rprintf(FINFO,"redoing %s(%d)\n",fname,i);
- SIVAL(buf, 0, i);
- send_msg(MSG_REDO, buf, 4);
+ SIVAL(numbuf, 0, i);
+ send_msg(MSG_REDO, numbuf, 4);
}
}
}
Index: rsync.1
--- rsync.1 15 Jan 2004 17:45:53 -0000 1.155
+++ rsync.1 17 Jan 2004 05:04:56 -0000
@@ -349,6 +349,7 @@ to the detailed description below for a
--delete delete files that don\&'t exist on sender
--delete-excluded also delete excluded files on receiver
--delete-after receiver deletes after transfer, not before
+ --delete-sent-files updated/sent files are removed from sender
--ignore-errors delete even if there are IO errors
--max-delete=NUM don\&'t delete more than NUM files
--partial keep partially transferred files
@@ -672,6 +673,12 @@ By default rsync does file deletions on
receiving side before transferring files to try to ensure that there is
sufficient space on the receiving filesystem\&. If you want to delete
after transferring, use the --delete-after switch\&. Implies --delete\&.
+.IP
+.IP "\fB--delete-sent-files\fP"
+This tells rsync to remove the source files
+on the sending side that are successfully transferred to the receiving
+side\&. Directories are not removed, nor are files that are identical on
+both systems\&.
.IP
.IP "\fB--ignore-errors\fP"
Tells --delete to go ahead and delete files
Index: rsync.h
--- rsync.h 15 Jan 2004 07:42:27 -0000 1.173
+++ rsync.h 17 Jan 2004 05:04:56 -0000
@@ -39,6 +39,7 @@
incompatible with older versions :-( */
#define CHAR_OFFSET 0
+/* These flags are used during the flist transfer. */
#define FLAG_DELETE (1<<0)
#define SAME_MODE (1<<1)
@@ -54,9 +55,14 @@
#define HAS_INODE_DATA (1<<9)
#define SAME_DEV (1<<10)
-/* What flags are relevant after the transfer of the flist is complete? */
+/* What flags above are relevant after the transfer of the flist? */
#define LIVE_FLAGS FLAG_DELETE
+/* These flist flags can be set after the flist is transferred. */
+
+/*#define FLAG_DELETE (1<<0) -- from the above list */
+#define FLAG_SENT (1<<1)
+
/* update this if you make incompatible changes */
#define PROTOCOL_VERSION 28
@@ -120,6 +126,7 @@ enum msgcode {
MSG_ERROR=FERROR, MSG_INFO=FINFO, MSG_LOG=FLOG, /* remote logging */
MSG_REDO=4, /* reprocess indicated flist index */
MSG_DONE=5, /* current phase is done */
+ MSG_SUCCESS=6, /* successfully updated indicated flist index */
};
#include "errcode.h"
Index: rsync.yo
--- rsync.yo 15 Jan 2004 17:45:53 -0000 1.139
+++ rsync.yo 17 Jan 2004 05:04:57 -0000
@@ -312,6 +312,7 @@ verb(
--delete delete files that don't exist on sender
--delete-excluded also delete excluded files on receiver
--delete-after receiver deletes after transfer, not before
+ --delete-sent-files updated/sent files are removed from sender
--ignore-errors delete even if there are IO errors
--max-delete=NUM don't delete more than NUM files
--partial keep partially transferred files
@@ -584,6 +585,11 @@ dit(bf(--delete-after)) By default rsync
receiving side before transferring files to try to ensure that there is
sufficient space on the receiving filesystem. If you want to delete
after transferring, use the --delete-after switch. Implies --delete.
+
+dit(bf(--delete-sent-files)) This tells rsync to remove the source files
+on the sending side that are successfully transferred to the receiving
+side. Directories are not removed, nor are files that are identical on
+both systems.
dit(bf(--ignore-errors)) Tells --delete to go ahead and delete files
even when there are IO errors.
Index: sender.c
--- sender.c 15 Jan 2004 08:56:33 -0000 1.34
+++ sender.c 17 Jan 2004 05:04:57 -0000
@@ -27,6 +27,7 @@ extern int dry_run;
extern int am_server;
extern int am_daemon;
extern int protocol_version;
+extern int delete_sent_files;
/**
@@ -104,7 +105,28 @@ static struct sum_struct *receive_sums(i
return s;
}
+static struct file_list *the_flist;
+void successful_send(int i)
+{
+ char fname[MAXPATHLEN];
+ struct file_struct *file;
+ int offset = 0;
+
+ if (!the_flist)
+ return;
+
+ file = the_flist->files[i];
+ if (!(file->flags & FLAG_SENT))
+ return; /* We didn't send it -- impossible! */
+ if (file->basedir) {
+ /* We know the name fits because we already sent it. */
+ offset = snprintf(fname, MAXPATHLEN, "%s/", file->basedir);
+ }
+ f_name_to(file, fname + offset, MAXPATHLEN - offset);
+ if (delete_sent_files && do_unlink(fname) == 0 && verbose > 0)
+ rprintf(FINFO, "sender removed %s\n", fname + offset);
+}
void send_files(struct file_list *flist, int f_out, int f_in)
{
@@ -129,6 +151,8 @@ void send_files(struct file_list *flist,
if (verbose > 2)
rprintf(FINFO, "send_files starting\n");
+ the_flist = flist;
+
while (1) {
int offset = 0;
@@ -307,6 +331,9 @@ void send_files(struct file_list *flist,
if (verbose > 2)
rprintf(FINFO, "sender finished %s\n", fname);
+
+ /* Flag that we actually sent this entry. */
+ file->flags |= FLAG_SENT;
}
if (verbose > 2)
More information about the rsync
mailing list