[SCM] The rsync repository. - branch master updated
Rsync CVS commit messages
rsync-cvs at lists.samba.org
Sat Apr 19 13:23:11 MDT 2014
The branch, master has been updated
via 22a3ac0 Add new-style compression that skips matching data.
from 1524c2e Expand the backslash description a bit more in excludes.
;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 22a3ac0b5538ec6c1ff222570413cbdb74fef67b
Author: Wayne Davison <wayned at samba.org>
Date: Sat Apr 19 12:11:11 2014 -0700
Add new-style compression that skips matching data.
Adding new-style compression that only compresses the literal data that
is sent over the wire and not also matching file data that was not sent.
This new-style compression is compatible with external zlib instances,
and will eventually become the default (once enough time has passed that
all servers support the --new-compress and --old-compress options).
NOTE: if you build rsync with an external zlib (i.e. if you specified
configure --with-included-zlib=no) you will ONLY get support for the
--new-compress option! A client will treat -z as uncompressed (with a
warning) and a server will exit with an error (unless -zz was used).
-----------------------------------------------------------------------
Summary of changes:
configure.ac | 1 +
options.c | 53 +++++++++++++++++++++++++----------
rsync.yo | 15 +++++++++-
testsuite/daemon-gzip-download.test | 2 +-
testsuite/daemon-gzip-upload.test | 2 +-
token.c | 38 ++++++++++++++----------
6 files changed, 77 insertions(+), 34 deletions(-)
Changeset truncated at 500 lines:
diff --git a/configure.ac b/configure.ac
index dccb2aa..c7b28c5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -792,6 +792,7 @@ if test x"$with_included_zlib" = x"yes"; then
BUILD_ZLIB='$(zlib_OBJS)'
CFLAGS="$CFLAGS -I$srcdir/zlib"
else
+ AC_DEFINE(EXTERNAL_ZLIB, 1, [Define to 1 if using external zlib])
AC_MSG_RESULT(no)
fi
diff --git a/options.c b/options.c
index dc9e62a..0aa64bf 100644
--- a/options.c
+++ b/options.c
@@ -32,6 +32,8 @@ extern unsigned int module_dirlen;
extern filter_rule_list filter_list;
extern filter_rule_list daemon_filter_list;
+#define NOT_SPECIFIED (-42)
+
int make_backups = 0;
/**
@@ -75,7 +77,7 @@ int protocol_version = PROTOCOL_VERSION;
int sparse_files = 0;
int preallocate_files = 0;
int do_compression = 0;
-int def_compress_level = Z_DEFAULT_COMPRESSION;
+int def_compress_level = NOT_SPECIFIED;
int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
int am_server = 0;
int am_sender = 0;
@@ -965,10 +967,12 @@ static struct poptOption long_options[] = {
{"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 },
{"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 },
+ {"old-compress", 0, POPT_ARG_VAL, &do_compression, 1, 0, 0 },
+ {"new-compress", 0, POPT_ARG_VAL, &do_compression, 2, 0, 0 },
{"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
{"no-z", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 },
{"skip-compress", 0, POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
- {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 'z', 0, 0 },
+ {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 0, 0, 0 },
{0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 },
{"progress", 0, POPT_ARG_VAL, &do_progress, 1, 0, 0 },
{"no-progress", 0, POPT_ARG_VAL, &do_progress, 0, 0, 0 },
@@ -1543,18 +1547,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
break;
case 'z':
- if (def_compress_level < Z_DEFAULT_COMPRESSION
- || def_compress_level > Z_BEST_COMPRESSION) {
- snprintf(err_buf, sizeof err_buf,
- "--compress-level value is invalid: %d\n",
- def_compress_level);
- return 0;
- }
- do_compression = def_compress_level != Z_NO_COMPRESSION;
- if (do_compression && refused_compress) {
- create_refuse_error(refused_compress);
- return 0;
- }
+ do_compression++;
break;
case 'M':
@@ -1829,6 +1822,33 @@ int parse_arguments(int *argc_p, const char ***argv_p)
exit_cleanup(0);
}
+ if (do_compression || def_compress_level != NOT_SPECIFIED) {
+ if (def_compress_level == NOT_SPECIFIED)
+ def_compress_level = Z_DEFAULT_COMPRESSION;
+ else if (def_compress_level < Z_DEFAULT_COMPRESSION || def_compress_level > Z_BEST_COMPRESSION) {
+ snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n",
+ def_compress_level);
+ return 0;
+ } else if (def_compress_level == Z_NO_COMPRESSION)
+ do_compression = 0;
+ else if (!do_compression)
+ do_compression = 1;
+ if (do_compression && refused_compress) {
+ create_refuse_error(refused_compress);
+ return 0;
+ }
+#ifdef EXTERNAL_ZLIB
+ if (do_compression == 1) {
+ snprintf(err_buf, sizeof err_buf,
+ "This rsync lacks old-style --compress due to its external zlib. Try -zz.\n");
+ if (am_server)
+ return 0;
+ fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf);
+ do_compression = 0;
+ }
+#endif
+ }
+
#ifdef HAVE_SETVBUF
if (outbuf_mode && !am_server) {
int mode = *(uchar *)outbuf_mode;
@@ -2451,7 +2471,7 @@ void server_options(char **args, int *argc_p)
}
if (sparse_files)
argstr[x++] = 'S';
- if (do_compression)
+ if (do_compression == 1)
argstr[x++] = 'z';
set_allow_inc_recurse();
@@ -2747,6 +2767,9 @@ void server_options(char **args, int *argc_p)
exit_cleanup(RERR_MALLOC);
}
+ if (do_compression > 1)
+ args[ac++] = "--new-compress";
+
if (remote_option_cnt) {
int j;
if (ac + remote_option_cnt > MAX_SERVER_ARGS) {
diff --git a/rsync.yo b/rsync.yo
index eafa385..c99d150 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -1867,7 +1867,20 @@ being transmitted -- something that is useful over a slow connection.
Note that this option typically achieves better compression ratios than can
be achieved by using a compressing remote shell or a compressing transport
because it takes advantage of the implicit information in the matching data
-blocks that are not explicitly sent over the connection.
+blocks that are not explicitly sent over the connection. This matching-data
+compression comes at a cost of CPU, though, and can be disabled by repeating
+the bf(-z) option, but only if both sides are at least version 3.1.1.
+
+Note that if your version of rsync was compiled with an external zlib (instead
+of the zlib that comes packaged with rsync) then it will not support the
+old-style compression, only the new-style (repeated-option) compression. In
+the future this new-style compression will likely become the default.
+
+The client rsync requests new-style compression on the server via the
+bf(--new-compress) option, so if you see that option rejected it means that
+the server is not new enough to support bf(-zz). Rsync also accepts the
+bf(--old-compress) option for a future time when new-style compression
+becomes the default.
See the bf(--skip-compress) option for the default list of file suffixes
that will not be compressed.
diff --git a/testsuite/daemon-gzip-download.test b/testsuite/daemon-gzip-download.test
index 89a112f..57dd820 100644
--- a/testsuite/daemon-gzip-download.test
+++ b/testsuite/daemon-gzip-download.test
@@ -31,7 +31,7 @@ hands_setup
# Build chkdir with a normal rsync and an --exclude.
$RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
-checkit "$RSYNC -avvvvz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
+checkit "$RSYNC -avvvvzz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
# The script would have aborted on error, so getting here means we've won.
exit 0
diff --git a/testsuite/daemon-gzip-upload.test b/testsuite/daemon-gzip-upload.test
index 11c52ba..b2110ea 100644
--- a/testsuite/daemon-gzip-upload.test
+++ b/testsuite/daemon-gzip-upload.test
@@ -25,7 +25,7 @@ hands_setup
# Build chkdir with a normal rsync and an --exclude.
$RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
-checkit "'$ignore23' $RSYNC -avvvvz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
+checkit "'$ignore23' $RSYNC -avvvvzz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
# The script would have aborted on error, so getting here means we've won.
exit 0
diff --git a/token.c b/token.c
index bd4d52f..8cc5532 100644
--- a/token.c
+++ b/token.c
@@ -23,10 +23,6 @@
#include "itypes.h"
#include <zlib.h>
-#ifndef Z_INSERT_ONLY
-#define Z_INSERT_ONLY Z_SYNC_FLUSH
-#endif
-
extern int do_compression;
extern int protocol_version;
extern int module_id;
@@ -406,9 +402,10 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
if (token == -1) {
/* end of file - clean up */
write_byte(f, END_FLAG);
- } else if (token != -2) {
+ } else if (token != -2 && do_compression == 1) {
/* Add the data in the current block to the compressor's
* history and hash table. */
+#ifndef EXTERNAL_ZLIB
do {
/* Break up long sections in the same way that
* see_deflate_token() does. */
@@ -418,17 +415,20 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
tx_strm.avail_in = n1;
if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */
offset += n1;
- do {
- tx_strm.next_out = (Bytef *) obuf;
- tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
- r = deflate(&tx_strm, Z_INSERT_ONLY);
- if (r != Z_OK) {
- rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
- r, tx_strm.avail_in);
- exit_cleanup(RERR_STREAMIO);
- }
- } while (tx_strm.avail_in != 0);
+ tx_strm.next_out = (Bytef *) obuf;
+ tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
+ r = deflate(&tx_strm, Z_INSERT_ONLY);
+ if (r != Z_OK || tx_strm.avail_in != 0) {
+ rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
+ r, tx_strm.avail_in);
+ exit_cleanup(RERR_STREAMIO);
+ }
} while (toklen > 0);
+#else
+ toklen++;
+ rprintf(FERROR, "Impossible error in external-zlib code (1).\n");
+ exit_cleanup(RERR_STREAMIO);
+#endif
}
}
@@ -579,6 +579,7 @@ static int32 recv_deflated_token(int f, char **data)
*/
static void see_deflate_token(char *buf, int32 len)
{
+#ifndef EXTERNAL_ZLIB
int r;
int32 blklen;
unsigned char hdr[5];
@@ -616,6 +617,11 @@ static void see_deflate_token(char *buf, int32 len)
exit_cleanup(RERR_STREAMIO);
}
} while (len || rx_strm.avail_out == 0);
+#else
+ buf++; len++;
+ rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
+ exit_cleanup(RERR_STREAMIO);
+#endif
}
/**
@@ -655,6 +661,6 @@ int32 recv_token(int f, char **data)
*/
void see_token(char *data, int32 toklen)
{
- if (do_compression)
+ if (do_compression == 1)
see_deflate_token(data, toklen);
}
--
The rsync repository.
More information about the rsync-cvs
mailing list