[SCM] The rsync repository. - branch master updated
Rsync CVS commit messages
rsync-cvs at lists.samba.org
Thu Apr 30 00:02:29 UTC 2020
The branch, master has been updated
via 3a7bf54a A resumed partial-dir file is transferred in-place. Fixed bug #13071.
from c1cb307b Fix a couple issues with the atime file-list value.
https://git.samba.org/?p=rsync.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 3a7bf54ad52072b36cecd7776d5d56612acd986f
Author: Wayne Davison <wayne at opencoder.net>
Date: Wed Apr 29 16:51:02 2020 -0700
A resumed partial-dir file is transferred in-place.
Fixed bug #13071.
-----------------------------------------------------------------------
Summary of changes:
NEWS | 8 ++++++--
compat.c | 6 ++++++
options.c | 1 +
receiver.c | 45 ++++++++++++++++++++++++++-------------------
rsync.yo | 9 ++++++++-
sender.c | 5 +++--
util.c | 5 +++++
7 files changed, 55 insertions(+), 24 deletions(-)
Changeset truncated at 500 lines:
diff --git a/NEWS b/NEWS
index 5e9d356f..d2043bae 100644
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,8 @@ Changes since 3.1.3:
- Some rrsync fixes and enhancements to handle the latest options.
+ - Fixed a crash in the --iconv code.
+
ENHANCEMENTS:
- Improved the --atimes patch and promoted it to be in the release.
@@ -49,11 +51,13 @@ Changes since 3.1.3:
rsync commands into/from untrusted directories (such as backups and
restores).
+ - When resuming the transfer of a file in the --partial-dir, rsync will now
+ update that partial file in-place instead of creating yet another tmp
+ file copy. This requires both sender & receiver to be at least v3.2.0.
+
- Added support for RSYNC_SHELL & RSYNC_NO_XFER_EXEC environment variables
that affect the pre-xfer exec and post-xfer exec rsync daemon options.
- - Fixed a crash in the --iconv code.
-
- Fixed a problem with the --link-dest|--copy-dest code when --xattrs was
specified along with multiple alternate-destination directories (it could
possibly choose a bad file match while trying to find a better xattr
diff --git a/compat.c b/compat.c
index 40de14bb..13a3b9f9 100644
--- a/compat.c
+++ b/compat.c
@@ -28,6 +28,7 @@ int compat_flags = 0;
int use_safe_inc_flist = 0;
int want_xattr_optim = 0;
int proper_seed_order = 0;
+int inplace_partial = 0;
extern int am_server;
extern int am_sender;
@@ -81,6 +82,7 @@ int filesfrom_convert = 0;
#define CF_SAFE_FLIST (1<<3)
#define CF_AVOID_XATTR_OPTIM (1<<4)
#define CF_CHKSUM_SEED_FIX (1<<5)
+#define CF_INPLACE_PARTIAL_DIR (1<<6)
static const char *client_info;
@@ -283,6 +285,8 @@ void setup_protocol(int f_out,int f_in)
compat_flags |= CF_AVOID_XATTR_OPTIM;
if (local_server || strchr(client_info, 'C') != NULL)
compat_flags |= CF_CHKSUM_SEED_FIX;
+ if (local_server || strchr(client_info, 'I') != NULL)
+ compat_flags |= CF_INPLACE_PARTIAL_DIR;
write_byte(f_out, compat_flags);
} else
compat_flags = read_byte(f_in);
@@ -313,6 +317,8 @@ void setup_protocol(int f_out,int f_in)
}
use_safe_inc_flist = (compat_flags & CF_SAFE_FLIST) || protocol_version >= 31;
need_messages_from_generator = 1;
+ if (compat_flags & CF_INPLACE_PARTIAL_DIR)
+ inplace_partial = 1;
#ifdef CAN_SET_SYMLINK_TIMES
} else if (!am_sender) {
receiver_symlink_times = 1;
diff --git a/options.c b/options.c
index 2319b14d..b2cbda53 100644
--- a/options.c
+++ b/options.c
@@ -2577,6 +2577,7 @@ void server_options(char **args, int *argc_p)
eFlags[x++] = 'f'; /* flist I/O-error safety support */
eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
eFlags[x++] = 'C'; /* support checksum seed order fix */
+ eFlags[x++] = 'I'; /* support inplace_partial behavior */
#undef eFlags
}
diff --git a/receiver.c b/receiver.c
index afc8932c..e95f1786 100644
--- a/receiver.c
+++ b/receiver.c
@@ -52,6 +52,7 @@ extern int keep_partial;
extern int checksum_seed;
extern int whole_file;
extern int inplace;
+extern int inplace_partial;
extern int allowed_lull;
extern int delay_updates;
extern int xfersum_type;
@@ -69,7 +70,7 @@ extern OFF_T preallocated_len;
static struct bitbag *delayed_bits = NULL;
static int phase = 0, redoing = 0;
static flist_ndx_list batch_redo_list;
-/* We're either updating the basis file or an identical copy: */
+/* This is non-0 when we are updating the basis file or an identical copy: */
static int updating_basis_or_equiv;
#define TMPNAME_SUFFIX ".XXXXXX"
@@ -233,7 +234,7 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file)
}
static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
- const char *fname, int fd, struct file_struct *file)
+ const char *fname, int fd, struct file_struct *file, int inplace_sizing)
{
static char file_sum1[MAX_DIGEST_LEN];
struct map_struct *mapbuf;
@@ -248,14 +249,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
char *map = NULL;
#ifdef SUPPORT_PREALLOCATION
- if (preallocate_files && fd != -1 && total_size > 0 && (!inplace || total_size > size_r)) {
+ if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) {
/* Try to preallocate enough space for file's eventual length. Can
* reduce fragmentation on filesystems like ext4, xfs, and NTFS. */
if ((preallocated_len = do_fallocate(fd, 0, total_size)) < 0)
rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
} else
#endif
- if (inplace) {
+ if (inplace_sizing) {
#ifdef HAVE_FTRUNCATE
/* The most compatible way to create a sparse file is to start with no length. */
if (sparse_files > 0 && whole_file && fd >= 0 && do_ftruncate(fd, 0) == 0)
@@ -382,9 +383,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
/* inplace: New data could be shorter than old data.
* preallocate_files: total_size could have been an overestimate.
* Cut off any extra preallocated zeros from dest file. */
- if ((inplace || preallocated_len > offset) && fd != -1 && !IS_DEVICE(file->mode) && do_ftruncate(fd, offset) < 0) {
- rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
- full_fname(fname));
+ if ((inplace_sizing || preallocated_len > offset) && fd != -1 && !IS_DEVICE(file->mode)) {
+ if (do_ftruncate(fd, offset) < 0)
+ rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", full_fname(fname));
}
#endif
@@ -407,7 +408,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
static void discard_receive_data(int f_in, struct file_struct *file)
{
- receive_data(f_in, NULL, -1, 0, NULL, -1, file);
+ receive_data(f_in, NULL, -1, 0, NULL, -1, file, 0);
}
static void handle_delayed_updates(char *local_name)
@@ -518,7 +519,7 @@ int recv_files(int f_in, int f_out, char *local_name)
int iflags, xlen;
char *fname, fbuf[MAXPATHLEN];
char xname[MAXPATHLEN];
- char fnametmp[MAXPATHLEN];
+ char *fnametmp, fnametmpbuf[MAXPATHLEN];
char *fnamecmp, *partialptr;
char fnamecmpbuf[MAXPATHLEN];
uchar fnamecmp_type;
@@ -530,7 +531,7 @@ int recv_files(int f_in, int f_out, char *local_name)
#ifdef SUPPORT_ACLS
const char *parent_dirname = "";
#endif
- int ndx, recv_ok;
+ int ndx, recv_ok, one_inplace;
if (DEBUG_GTE(RECV, 1))
rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used);
@@ -752,6 +753,7 @@ int recv_files(int f_in, int f_out, char *local_name)
if (fd1 == -1 && protocol_version < 29) {
if (fnamecmp != fname) {
fnamecmp = fname;
+ fnamecmp_type = FNAMECMP_FNAME;
fd1 = do_open(fnamecmp, O_RDONLY, 0);
}
@@ -760,12 +762,14 @@ int recv_files(int f_in, int f_out, char *local_name)
pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
basis_dir[0], fname);
fnamecmp = fnamecmpbuf;
+ fnamecmp_type = FNAMECMP_BASIS_DIR_LOW;
fd1 = do_open(fnamecmp, O_RDONLY, 0);
}
}
- updating_basis_or_equiv = inplace
- && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP);
+ one_inplace = inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR;
+ updating_basis_or_equiv = one_inplace
+ || (inplace && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP));
if (fd1 == -1) {
st.st_mode = 0;
@@ -831,14 +835,16 @@ int recv_files(int f_in, int f_out, char *local_name)
}
/* We now check to see if we are writing the file "inplace" */
- if (inplace) {
- fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600);
+ if (inplace || one_inplace) {
+ fnametmp = one_inplace ? partialptr : fname;
+ fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600);
if (fd2 == -1) {
rsyserr(FERROR_XFER, errno, "open %s failed",
- full_fname(fname));
+ full_fname(fnametmp));
} else if (updating_basis_or_equiv)
cleanup_set(NULL, NULL, file, fd1, fd2);
} else {
+ fnametmp = fnametmpbuf;
fd2 = open_tmpfile(fnametmp, fname, file);
if (fd2 != -1)
cleanup_set(fnametmp, partialptr, file, fd1, fd2);
@@ -860,7 +866,7 @@ int recv_files(int f_in, int f_out, char *local_name)
rprintf(FINFO, "%s\n", fname);
/* recv file data */
- recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file);
+ recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file, inplace || one_inplace);
log_item(log_code, file, iflags, NULL);
if (want_progress_now)
@@ -881,10 +887,11 @@ int recv_files(int f_in, int f_out, char *local_name)
partialptr, file, recv_ok, 1))
recv_ok = -1;
else if (fnamecmp == partialptr) {
- do_unlink(partialptr);
+ if (!one_inplace)
+ do_unlink(partialptr);
handle_partial_dir(partialptr, PDIR_DELETE);
}
- } else if (keep_partial && partialptr) {
+ } else if (keep_partial && partialptr && !one_inplace) {
if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
rprintf(FERROR,
"Unable to create partial-dir for %s -- discarding %s.\n",
@@ -900,7 +907,7 @@ int recv_files(int f_in, int f_out, char *local_name)
recv_ok = 2;
} else
partialptr = NULL;
- } else
+ } else if (!one_inplace)
do_unlink(fnametmp);
cleanup_disable();
diff --git a/rsync.yo b/rsync.yo
index 59d84265..b1e4ec3d 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -2459,7 +2459,9 @@ Rsync will create the em(DIR) if it is missing (just the last dir -- not
the whole path). This makes it easy to use a relative path (such as
"bf(--partial-dir=.rsync-partial)") to have rsync create the
partial-directory in the destination file's directory when needed, and then
-remove it again when the partial file is deleted.
+remove it again when the partial file is deleted. Note that the directory
+is only removed if it is a relative pathname, as it is expected that an
+absolute path is to a directory that is reserved for partial-dir work.
If the partial-dir value is not an absolute path, rsync will add an exclude
rule at the end of all your existing excludes. This will prevent the
@@ -2493,6 +2495,11 @@ option does not look for this environment value are (1) when bf(--inplace) was
specified (since bf(--inplace) conflicts with bf(--partial-dir)), and (2) when
bf(--delay-updates) was specified (see below).
+When a modern rsync resumes the transfer of a file in the partial-dir, that
+partial file is now updated in-place instead of creating yet another tmp-file
+copy (so it maxes out at dest + tmp instead of dest + partial + tmp). This
+requires both ends of the transfer to be at least version 3.2.0.
+
For the purposes of the daemon-config's "refuse options" setting,
bf(--partial-dir) does em(not) imply bf(--partial). This is so that a
refusal of the bf(--partial) option can be used to disallow the overwriting
diff --git a/sender.c b/sender.c
index 4f34cded..51feec72 100644
--- a/sender.c
+++ b/sender.c
@@ -42,6 +42,7 @@ extern int remove_source_files;
extern int updating_basis_file;
extern int make_backups;
extern int inplace;
+extern int inplace_partial;
extern int batch_fd;
extern int write_batch;
extern int file_old_total;
@@ -316,8 +317,8 @@ void send_files(int f_in, int f_out)
stats.created_files++;
}
- updating_basis_file = inplace && (protocol_version >= 29
- ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0);
+ updating_basis_file = (inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR)
+ || (inplace && (protocol_version >= 29 ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0));
if (!am_server)
set_current_file_index(file, ndx);
diff --git a/util.c b/util.c
index 03917c31..edb3ae03 100644
--- a/util.c
+++ b/util.c
@@ -498,6 +498,11 @@ int robust_rename(const char *from, const char *to, const char *partialptr,
{
int tries = 4;
+ /* A resumed in-place partial-dir transfer might call us with from and
+ * to pointing to the same buf if the transfer failed yet again. */
+ if (from == to)
+ return 0;
+
while (tries--) {
if (do_rename(from, to) == 0)
return 0;
--
The rsync repository.
More information about the rsync-cvs
mailing list