[SCM] The rsync repository. - branch master updated
Rsync CVS commit messages
rsync-cvs at lists.samba.org
Fri May 22 16:54:38 UTC 2020
The branch, master has been updated
via 4f6c8c66 Checksum negotiation & more bits for compat_flags
via a7303a3d Fix a bug in the writing of the batch.sh file
from 70c6b408 SSE2/SSSE3 optimized version of get_checksum1() for x86-64
https://git.samba.org/?p=rsync.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 4f6c8c6652c74b958c631ca9a16f450c6ce6a23c
Author: Wayne Davison <wayne at opencoder.net>
Date: Fri May 22 08:23:26 2020 -0700
Checksum negotiation & more bits for compat_flags
- Add checksum negotiation to the protocol so that we can easily add new
checksum algorithms and each will be used when both sides support it.
- Increase the size of the compat_flags value in the protocol from a
byte to an int.
commit a7303a3d3d1ab3221bcb950f19eefb10e27b4bf7
Author: Wayne Davison <wayne at opencoder.net>
Date: Fri May 22 08:10:41 2020 -0700
Fix a bug in the writing of the batch.sh file
Fix the code that writes the options and the default destination path
into the batch.sh file to be able to handle options being specified
after source/dest args.
-----------------------------------------------------------------------
Summary of changes:
batch.c | 15 ++++++++---
checksum.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++------------
compat.c | 44 +++++++++++++++++++++++++++----
io.c | 3 ++-
main.c | 2 +-
options.c | 8 +++---
rsync.yo | 45 ++++++++++++++++++++++++--------
7 files changed, 163 insertions(+), 42 deletions(-)
Changeset truncated at 500 lines:
diff --git a/batch.c b/batch.c
index 263a9a35..8915d9e0 100644
--- a/batch.c
+++ b/batch.c
@@ -213,7 +213,7 @@ static void write_filter_rules(int fd)
* understand most of the options, so it uses some overly simple
* heuristics to munge the command line into something that will
* (hopefully) work. */
-void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt)
+void write_batch_shell_file(int argc, char *argv[], int file_argc, char *file_argv[])
{
int fd, i, len, err = 0;
char *p, *p2, filename[MAXPATHLEN];
@@ -237,8 +237,15 @@ void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt)
else
write_sbuf(fd, " --exclude-from=-");
}
- for (i = 1; i < argc - file_arg_cnt; i++) {
+ for (i = 1; i < argc; i++) {
p = argv[i];
+ if (file_argc && p == file_argv[0]) {
+ if (file_argc > 1) {
+ file_argv++;
+ file_argc--;
+ }
+ continue;
+ }
if (strncmp(p, "--files-from", 12) == 0
|| strncmp(p, "--filter", 8) == 0
|| strncmp(p, "--include", 9) == 0
@@ -267,8 +274,8 @@ void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt)
err = 1;
}
}
- if (!(p = check_for_hostspec(argv[argc - 1], &p2, &i)))
- p = argv[argc - 1];
+ if (!(p = check_for_hostspec(file_argv[file_argc - 1], &p2, &i)))
+ p = file_argv[file_argc - 1];
if (write(fd, " ${1:-", 6) != 6
|| write_arg(fd, p) < 0)
err = 1;
diff --git a/checksum.c b/checksum.c
index 8698543d..980d262b 100644
--- a/checksum.c
+++ b/checksum.c
@@ -21,6 +21,10 @@
#include "rsync.h"
+extern int am_server;
+extern int local_server;
+extern int whole_file;
+extern int read_batch;
extern int checksum_seed;
extern int protocol_version;
extern int proper_seed_order;
@@ -33,27 +37,21 @@ extern char *checksum_choice;
#define CSUM_MD4 4
#define CSUM_MD5 5
+const char *default_checksum_list =
+ "md5 md4";
+
+#define MAX_CHECKSUM_LIST 1024
+
int xfersum_type = 0; /* used for the file transfer checksums */
int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */
+const char *negotiated_csum_name = NULL;
-/* Returns 1 if --whole-file must be enabled. */
-int parse_checksum_choice(void)
-{
- char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
- if (cp) {
- xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice);
- checksum_type = parse_csum_name(cp+1, -1);
- } else
- xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
- return xfersum_type == CSUM_NONE;
-}
-
-int parse_csum_name(const char *name, int len)
+static int parse_csum_name(const char *name, int len, int allow_auto)
{
if (len < 0 && name)
len = strlen(name);
- if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) {
+ if (!name || (allow_auto && len == 4 && strncasecmp(name, "auto", 4) == 0)) {
if (protocol_version >= 30)
return CSUM_MD5;
if (protocol_version >= 27)
@@ -69,7 +67,65 @@ int parse_csum_name(const char *name, int len)
if (len == 4 && strncasecmp(name, "none", 4) == 0)
return CSUM_NONE;
- rprintf(FERROR, "unknown checksum name: %s\n", name);
+ if (allow_auto) {
+ rprintf(FERROR, "unknown checksum name: %s\n", name);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+
+ return -1;
+}
+
+void parse_checksum_choice(void)
+{
+ if (!negotiated_csum_name) {
+ char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
+ if (cp) {
+ xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice, 1);
+ checksum_type = parse_csum_name(cp+1, -1, 1);
+ } else
+ xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1, 1);
+ }
+ if (xfersum_type == CSUM_NONE)
+ whole_file = 1;
+}
+
+void negotiate_checksum(int f_in, int f_out, const char *csum_list)
+{
+ char *tok, sumbuf[MAX_CHECKSUM_LIST];
+ int sum_type, len;
+
+ if (!am_server || local_server) {
+ if (!csum_list || !*csum_list)
+ csum_list = default_checksum_list;
+ len = strlen(csum_list);
+ if (len >= (int)sizeof sumbuf) {
+ rprintf(FERROR, "The checksum list is too long.\n");
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+ if (!local_server)
+ write_vstring(f_out, csum_list, len);
+ }
+
+ if (local_server && !read_batch)
+ memcpy(sumbuf, csum_list, len+1);
+ else
+ len = read_vstring(f_in, sumbuf, sizeof sumbuf);
+
+ if (len > 0) {
+ for (tok = strtok(sumbuf, " \t"); tok; tok = strtok(NULL, " \t")) {
+ len = strlen(tok);
+ sum_type = parse_csum_name(tok, len, 0);
+ if (sum_type >= CSUM_NONE) {
+ xfersum_type = checksum_type = sum_type;
+ if (am_server && !local_server)
+ write_vstring(f_out, tok, len);
+ negotiated_csum_name = strdup(tok);
+ return;
+ }
+ }
+ }
+
+ rprintf(FERROR, "Failed to negotiate a common checksum\n");
exit_cleanup(RERR_UNSUPPORTED);
}
@@ -260,7 +316,7 @@ void sum_init(int csum_type, int seed)
char s[4];
if (csum_type < 0)
- csum_type = parse_csum_name(NULL, 0);
+ csum_type = parse_csum_name(NULL, 0, 1);
cursum_type = csum_type;
switch (csum_type) {
diff --git a/compat.c b/compat.c
index bd313fa9..b29b9637 100644
--- a/compat.c
+++ b/compat.c
@@ -41,6 +41,7 @@ extern int preallocate_files;
extern int append_mode;
extern int fuzzy_basis;
extern int read_batch;
+extern int write_batch;
extern int delay_updates;
extern int checksum_seed;
extern int basis_dir_cnt;
@@ -60,12 +61,14 @@ extern char *partial_dir;
extern char *dest_option;
extern char *files_from;
extern char *filesfrom_host;
+extern char *checksum_choice;
extern filter_rule_list filter_list;
extern int need_unsorted_flist;
#ifdef ICONV_OPTION
extern iconv_t ic_send, ic_recv;
extern char *iconv_opt;
#endif
+extern const char *negotiated_csum_name;
/* These index values are for the file-list's extra-attribute array. */
int pathname_ndx, depth_ndx, atimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
@@ -141,6 +144,8 @@ void set_allow_inc_recurse(void)
void setup_protocol(int f_out,int f_in)
{
+ int csum_exchange = 0;
+
assert(file_extra_cnt == 0);
assert(EXTRA64_CNT == 2 || EXTRA64_CNT == 1);
@@ -289,11 +294,23 @@ void setup_protocol(int f_out,int f_in)
compat_flags |= CF_CHKSUM_SEED_FIX;
if (local_server || strchr(client_info, 'I') != NULL)
compat_flags |= CF_INPLACE_PARTIAL_DIR;
- if (local_server || strchr(client_info, 'V') != NULL)
- compat_flags |= CF_VARINT_FLIST_FLAGS;
- write_byte(f_out, compat_flags);
- } else
- compat_flags = read_byte(f_in);
+ if (local_server || strchr(client_info, 'v') != NULL) {
+ if (!write_batch || protocol_version >= 30) {
+ csum_exchange = 1;
+ compat_flags |= CF_VARINT_FLIST_FLAGS;
+ }
+ }
+ if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */
+ if (!write_batch)
+ compat_flags |= CF_VARINT_FLIST_FLAGS;
+ write_byte(f_out, compat_flags);
+ } else
+ write_varint(f_out, compat_flags);
+ } else { /* read_varint() is compatible with the older write_byte() when the 0x80 bit isn't on. */
+ compat_flags = read_varint(f_in);
+ if (compat_flags & CF_VARINT_FLIST_FLAGS)
+ csum_exchange = 1;
+ }
/* The inc_recurse var MUST be set to 0 or 1. */
inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0;
want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM);
@@ -358,5 +375,22 @@ void setup_protocol(int f_out,int f_in)
checksum_seed = read_int(f_in);
}
+ if (!checksum_choice) {
+ const char *rcl = getenv("RSYNC_CHECKSUM_LIST");
+ if (csum_exchange)
+ negotiate_checksum(f_in, f_out, rcl);
+ else if (!am_server && rcl && *rcl && strstr(rcl, "FAIL")) {
+ rprintf(FERROR, "Remote rsync is too old for checksum negotation\n");
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+ }
+
init_flist();
}
+
+void maybe_write_checksum(int batch_fd)
+{
+ assert(negotiated_csum_name != NULL);
+ if (compat_flags & CF_VARINT_FLIST_FLAGS)
+ write_vstring(batch_fd, negotiated_csum_name, strlen(negotiated_csum_name));
+}
diff --git a/io.c b/io.c
index c6d2023c..446a5f34 100644
--- a/io.c
+++ b/io.c
@@ -2368,8 +2368,9 @@ void start_write_batch(int fd)
* is involved. */
write_int(batch_fd, protocol_version);
if (protocol_version >= 30)
- write_byte(batch_fd, compat_flags);
+ write_varint(batch_fd, compat_flags);
write_int(batch_fd, checksum_seed);
+ maybe_write_checksum(batch_fd);
if (am_sender)
write_batch_monitor_out = fd;
diff --git a/main.c b/main.c
index ca96270a..2e2094d2 100644
--- a/main.c
+++ b/main.c
@@ -1726,7 +1726,7 @@ int main(int argc,char *argv[])
if ((write_batch || read_batch) && !am_server) {
if (write_batch)
- write_batch_shell_file(orig_argc, orig_argv, argc);
+ write_batch_shell_file(orig_argc, orig_argv, argc, argv);
if (read_batch && strcmp(batch_name, "-") == 0)
batch_fd = STDIN_FILENO;
diff --git a/options.c b/options.c
index ca3b97e1..c1e957b8 100644
--- a/options.c
+++ b/options.c
@@ -1928,12 +1928,11 @@ int parse_arguments(int *argc_p, const char ***argv_p)
}
}
- if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) {
+ if (checksum_choice && strcasecmp(checksum_choice, "auto") != 0 && strcasecmp(checksum_choice, "auto,auto") != 0) {
/* Call this early to verify the args and figure out if we need to force
* --whole-file. Note that the parse function will get called again later,
* just in case an "auto" choice needs to know the protocol_version. */
- if (parse_checksum_choice())
- whole_file = 1;
+ parse_checksum_choice();
} else
checksum_choice = NULL;
@@ -2642,7 +2641,8 @@ void server_options(char **args, int *argc_p)
eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
eFlags[x++] = 'C'; /* support checksum seed order fix */
eFlags[x++] = 'I'; /* support inplace_partial behavior */
- eFlags[x++] = 'V'; /* use varint for flist flags */
+ eFlags[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */
+ /* NOTE: Avoid using 'V' -- it was the high bit of a write_byte() that became write_varint(). */
#undef eFlags
}
diff --git a/rsync.yo b/rsync.yo
index ed774968..a1917cf6 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -657,8 +657,9 @@ checksum that is generated as the file is transferred, but that
automatic after-the-transfer verification has nothing to do with this
option's before-the-transfer "Does this file need to be updated?" check.
-For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
-MD5. For older protocols, the checksum used is MD4.
+The checksum used is auto-negotiated between the client and the server, but
+can be overridden using either the bf(--checksum-choice) option or an
+environment variable that is discussed in that option's section.
dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
way of saying you want recursion and want to preserve almost
@@ -1371,16 +1372,38 @@ batch-writing option is in effect.
dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms.
If one algorithm name is specified, it is used for both the transfer checksums
-and (assuming bf(--checksum) is specified) the pre-transfer checksumming. If two
+and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two
comma-separated names are supplied, the first name affects the transfer
-checksums, and the second name affects the pre-transfer checksumming.
-
-The algorithm choices are "auto", "md4", "md5", and "none". If "none" is
-specified for the first name, the bf(--whole-file) option is forced on and no
-checksum verification is performed on the transferred data. If "none" is
-specified for the second name, the bf(--checksum) option cannot be used. The
-"auto" option is the default, where rsync bases its algorithm choice on the
-protocol version (for backward compatibility with older rsync versions).
+checksums, and the second name affects the pre-transfer checksums (bf(-c)).
+
+The algorithm choices are "auto", "MD5", "MD4", and "none".
+
+If "none" is specified for the first (or only) name, the bf(--whole-file) option
+is forced on and no checksum verification is performed on the transferred data.
+If "none" is specified for the second (or only) name, the bf(--checksum) option
+cannot be used.
+
+The "auto" option is the default, where rsync bases its algorithm choice on a
+negotation between the client and the server as follows:
+
+If both the client and the server are at least version 3.2.0, they will
+exchange a list of checksum names and choose the first one in the list that
+they have in common.
+This typically means that they will choose MD5.
+If one side of the transfer is not new enough to support this checksum
+negotation, then a value is chosen based on the protocol version (which
+chooses between MD5 and various flavors of MD4 based on protocol age).
+
+You can also override the checksum using the RSYNC_CHECKSUM_LIST environment
+variable by setting it to a space-separated list of checksum names that you
+consider acceptable. If no common checksum is found, the client exits with an
+error. This method does not allow you to specify the transfer checksum
+separately from the pre-transfer checksum, and it ignores "auto" and all
+unknown checksum names. If the remote rsync is not new enough to handle a
+checksum negotiation list, the list is silently ignored unless it contains the
+string "FAIL" in it.
+
+The use of the bf(--checksum-choice) option overrides this environment list.
dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a
filesystem boundary when recursing. This does not limit the user's ability
--
The rsync repository.
More information about the rsync-cvs
mailing list