[SCM] The rsync repository. - branch master updated
Rsync CVS commit messages
rsync-cvs at lists.samba.org
Sun Mar 13 18:31:43 UTC 2022
The branch, master has been updated
via 13c4019e Also ignore a root-level rrsync file.
via b7b387b1 Add FALLTHROUGH comment.
via 7569edfa Use ac_includes_default in largefile support test.
via 55b2a068 Test newer FreeBSD.
via b81a5095 Make asm use more selectable
from 26f4dbe1 Change usage (--version) output to note when ASM isn't really being used.
https://git.samba.org/?p=rsync.git;a=shortlog;h=master
- Log -----------------------------------------------------------------
commit 13c4019e94015b234697c75d9d3624862e962d3c
Author: Wayne Davison <wayne at opencoder.net>
Date: Sun Mar 13 10:45:09 2022 -0700
Also ignore a root-level rrsync file.
commit b7b387b1f798f675109986eaf3e62fdf41037c79
Author: Wayne Davison <wayne at opencoder.net>
Date: Sun Mar 13 09:31:44 2022 -0700
Add FALLTHROUGH comment.
commit 7569edfaef46e1b631849c884c7c4937ba425a0f
Author: Wayne Davison <wayne at opencoder.net>
Date: Wed Mar 9 18:38:03 2022 -0800
Use ac_includes_default in largefile support test.
commit 55b2a06812e39f15f52e92b979600cb82be210cb
Author: Wayne Davison <wayne at opencoder.net>
Date: Thu Mar 3 17:26:47 2022 -0800
Test newer FreeBSD.
commit b81a5095563776397a4239132d2b737a1083e02f
Author: Wayne Davison <wayne at opencoder.net>
Date: Thu Mar 3 17:00:57 2022 -0800
Make asm use more selectable
- Make the SIMD ASM code off by default. Use configure --enable-simd-asm
to enable.
- Allow MD5 ASM code to be requested even when OpenSSL is handling MD4
checksums. Use configure --enable-md5-asm to enable.
-----------------------------------------------------------------------
Summary of changes:
.cirrus.yml | 2 +-
.gitignore | 1 +
Makefile.in | 15 ++---
NEWS.md | 21 +++++--
checksum.c | 34 +++++------
configure.ac | 111 +++++++++++++++++++---------------
lib/md5-asm-x86_64.S | 4 +-
lib/md5.c | 19 +++---
lib/mdigest.h | 13 ++--
rsync.h | 9 +--
simd-checksum-avx2.S | 14 ++++-
simd-checksum-x86_64.cpp | 151 +++++++++++++++++++++++++++++++++++++++++++----
syscall.c | 2 +-
usage.c | 13 ++--
14 files changed, 292 insertions(+), 117 deletions(-)
Changeset truncated at 500 lines:
diff --git a/.cirrus.yml b/.cirrus.yml
index 5112ef88..16489f8f 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,7 +1,7 @@
freebsd_task:
name: FreeBSD
freebsd_instance:
- image_family: freebsd-12-2
+ image_family: freebsd-13-0
env:
PATH: /usr/local/bin:$PATH
prep_script:
diff --git a/.gitignore b/.gitignore
index 85290116..9e59c9c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ aclocal.m4
/proto.h
/proto.h-tstamp
/rsync*.[15]
+/rrsync
/rrsync*.1
/rsync*.html
/rrsync*.html
diff --git a/Makefile.in b/Makefile.in
index 8817edab..3cde9557 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -30,8 +30,9 @@ SHELL=/bin/sh
.SUFFIXES:
.SUFFIXES: .c .o
-SIMD_x86_64=simd-checksum-x86_64.o simd-checksum-avx2.o
-ASM_x86_64=lib/md5-asm-x86_64.o
+ROLL_SIMD_x86_64=simd-checksum-x86_64.o
+ROLL_ASM_x86_64=simd-checksum-avx2.o
+MD5_ASM_x86_64=lib/md5-asm-x86_64.o
GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html \
@@ -46,7 +47,7 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
-OBJS3=progress.o pipe.o @ASM@ @SIMD@
+OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
@@ -147,13 +148,13 @@ git-version.h: ALWAYS_RUN
ALWAYS_RUN:
simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
- @$(srcdir)/cmd-or-msg disable-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
+ @$(srcdir)/cmd-or-msg disable-roll-simd $(CXX) -I. $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp
simd-checksum-avx2.o: simd-checksum-avx2.S
- @$(srcdir)/cmd-or-msg disable-asm $(CC) $(CFLAGS) --include=$(srcdir)/rsync.h -DAVX2_ASM -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
+ @$(srcdir)/cmd-or-msg disable-roll-asm $(CC) $(CFLAGS) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/simd-checksum-avx2.S
-lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S config.h lib/md-defines.h
- @$(srcdir)/cmd-or-msg disable-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
+lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.S lib/md-defines.h
+ @$(srcdir)/cmd-or-msg disable-md5-asm $(CC) -I. @NOEXECSTACK@ -c -o $@ $(srcdir)/lib/md5-asm-x86_64.S
tls$(EXEEXT): $(TLS_OBJ)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS)
diff --git a/NEWS.md b/NEWS.md
index 87abba21..8e47ca37 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -136,7 +136,8 @@
(keeping the behavior the same as before), so specifying `--info=nonreg0`
can be used to turn the warnings off.
- - More ASM optimizations from Shark64.
+ - An optional asm optimization for the rolling checksum from Shark64. Enable
+ it with `./configure --enable-roll-asm`.
- Using `--debug=FILTER` now outputs a caution message if a filter rule
has trailing whitespace.
@@ -192,14 +193,24 @@
using the output of `git describe` when building inside a non-shallow git
checkout, though.)
- - Improved the IPv6 determination in configure.
+ - Renamed configure's `--enable-simd` option to `--enable-roll-simd` and added
+ the option `--enable-roll-asm` to use the new asm version of the code. Both
+ are x86_64/amd64 only.
+
+ - Renamed configure's `--enable-asm` option to `--enable-md5-asm` to avoid
+ confusion with the asm option for the rolling checksum. It is also honored
+ even when openssl crypto is in use. This allows: normal MD4 & MD5, normal
+ MD4 + asm MD5, openssl MD4 & MD5, or openssl MD4 + asm MD5.
- - Made SIMD & ASM configure default to "no" on non-Linux hosts due to various
- reports of problems on NetBSD & macOS hosts. These tests were also tweaked
- to allow enabling the feature on a host_cpu of amd64 (was only x86_64).
+ - Made SIMD & asm configure checks default to "no" on non-Linux hosts due to
+ various reports of problems on NetBSD & macOS hosts. These were also
+ tweaked to allow enabling the feature on a host_cpu of amd64 (was only
+ allowed on x86_64 before).
- Fixed configure to not fail at the SIMD check when cross-compiling.
+ - Improved the IPv6 determination in configure.
+
- Compile the C files with `-pedantic-errors` (when possible) so that we will
get warned if a static initialization overflows in the future (among other
things).
diff --git a/checksum.c b/checksum.c
index 1ed76828..77848585 100644
--- a/checksum.c
+++ b/checksum.c
@@ -179,7 +179,7 @@ int canonical_checksum(int csum_type)
return 0;
}
-#ifndef HAVE_SIMD /* See simd-checksum-*.cpp. */
+#ifndef USE_ROLL_SIMD /* See simd-checksum-*.cpp. */
/*
a simple 32 bit checksum that can be updated from either end
(inspired by Mark Adler's Adler-32 checksum)
@@ -222,23 +222,23 @@ void get_checksum2(char *buf, int32 len, char *sum)
}
#endif
case CSUM_MD5: {
- MD5_CTX m5;
+ md5_context m5;
uchar seedbuf[4];
- MD5_Init(&m5);
+ md5_begin(&m5);
if (proper_seed_order) {
if (checksum_seed) {
SIVALu(seedbuf, 0, checksum_seed);
- MD5_Update(&m5, seedbuf, 4);
+ md5_update(&m5, seedbuf, 4);
}
- MD5_Update(&m5, (uchar *)buf, len);
+ md5_update(&m5, (uchar *)buf, len);
} else {
- MD5_Update(&m5, (uchar *)buf, len);
+ md5_update(&m5, (uchar *)buf, len);
if (checksum_seed) {
SIVALu(seedbuf, 0, checksum_seed);
- MD5_Update(&m5, seedbuf, 4);
+ md5_update(&m5, seedbuf, 4);
}
}
- MD5_Final((uchar *)sum, &m5);
+ md5_result(&m5, (uchar *)sum);
break;
}
case CSUM_MD4:
@@ -374,18 +374,18 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
}
#endif
case CSUM_MD5: {
- MD5_CTX m5;
+ md5_context m5;
- MD5_Init(&m5);
+ md5_begin(&m5);
for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
- MD5_Update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+ md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
remainder = (int32)(len - i);
if (remainder > 0)
- MD5_Update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
+ md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
- MD5_Final((uchar *)sum, &m5);
+ md5_result(&m5, (uchar *)sum);
break;
}
case CSUM_MD4:
@@ -443,7 +443,7 @@ static union {
#ifdef USE_OPENSSL
MD4_CTX m4;
#endif
- MD5_CTX m5;
+ md5_context m5;
} ctx;
#ifdef SUPPORT_XXHASH
static XXH64_state_t* xxh64_state;
@@ -482,7 +482,7 @@ void sum_init(int csum_type, int seed)
break;
#endif
case CSUM_MD5:
- MD5_Init(&ctx.m5);
+ md5_begin(&ctx.m5);
break;
case CSUM_MD4:
#ifdef USE_OPENSSL
@@ -532,7 +532,7 @@ void sum_update(const char *p, int32 len)
break;
#endif
case CSUM_MD5:
- MD5_Update(&ctx.m5, (uchar *)p, len);
+ md5_update(&ctx.m5, (uchar *)p, len);
break;
case CSUM_MD4:
#ifdef USE_OPENSSL
@@ -597,7 +597,7 @@ int sum_end(char *sum)
}
#endif
case CSUM_MD5:
- MD5_Final((uchar *)sum, &ctx.m5);
+ md5_result(&ctx.m5, (uchar *)sum);
break;
case CSUM_MD4:
#ifdef USE_OPENSSL
diff --git a/configure.ac b/configure.ac
index 9960fafc..f23c6a93 100644
--- a/configure.ac
+++ b/configure.ac
@@ -229,12 +229,12 @@ fi
AC_DEFINE_UNQUOTED(NOBODY_USER, "$NOBODY_USER", [unprivileged user--e.g. nobody])
AC_DEFINE_UNQUOTED(NOBODY_GROUP, "$NOBODY_GROUP", [unprivileged group for unprivileged user])
-# SIMD optimizations
-SIMD=
+# rolling-checksum SIMD optimizations
+ROLL_SIMD=
-AC_MSG_CHECKING([whether to enable SIMD optimizations])
-AC_ARG_ENABLE(simd,
- AS_HELP_STRING([--enable-simd],[enable/disable to control SIMD optimizations (requires c++)]))
+AC_MSG_CHECKING([whether to enable rolling-checksum SIMD optimizations])
+AC_ARG_ENABLE(roll-simd,
+ AS_HELP_STRING([--enable-roll-simd],[enable/disable to control rolling-checksum SIMD optimizations (requires c++)]))
# Clag is crashing with -g -O2, so we'll get rid of -g for now.
CXXFLAGS=`echo "$CXXFLAGS" | sed 's/-g //'`
@@ -263,14 +263,14 @@ __attribute__ ((target("ssse3"))) void more_testing(char* buf, int len)
}
]])
-if test x"$enable_simd" = x""; then
+if test x"$enable_roll_simd" = x""; then
case "$host_os" in
*linux*) ;;
- *) enable_simd=no ;;
+ *) enable_roll_simd=no ;;
esac
fi
-if test x"$enable_simd" != x"no"; then
+if test x"$enable_roll_simd" != x"no"; then
# For x86-64 SIMD, g++ >=5 or clang++ >=7 is required
if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
AC_LANG(C++)
@@ -283,23 +283,23 @@ if test x"$enable_simd" != x"no"; then
AC_LANG(C)
if test x"$CXX_OK" = x"yes"; then
# AC_MSG_RESULT() is called below.
- SIMD="$host_cpu"
- elif test x"$enable_simd" = x"yes"; then
+ ROLL_SIMD="$host_cpu"
+ elif test x"$enable_roll_simd" = x"yes"; then
AC_MSG_RESULT(error)
- AC_MSG_ERROR(The SIMD compilation test failed.
-Omit --enable-simd to continue without it.)
+ AC_MSG_ERROR(The rolling-checksum SIMD compilation test failed.
+Omit --enable-roll-simd to continue without it.)
fi
- elif test x"$enable_simd" = x"yes"; then
+ elif test x"$enable_roll_simd" = x"yes"; then
AC_MSG_RESULT(unavailable)
- AC_MSG_ERROR(The SIMD optimizations are currently x86_64|amd64 only.
-Omit --enable-simd to continue without it.)
+ AC_MSG_ERROR(The rolling-checksum SIMD optimizations are currently x86_64|amd64 only.
+Omit --enable-roll-simd to continue without it.)
fi
fi
-if test x"$SIMD" != x""; then
- AC_MSG_RESULT([yes ($SIMD)])
- AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations])
- SIMD='$(SIMD_'"$SIMD)"
+if test x"$ROLL_SIMD" != x""; then
+ AC_MSG_RESULT([yes ($ROLL_SIMD)])
+ AC_DEFINE(USE_ROLL_SIMD, 1, [Define to 1 to enable rolling-checksum SIMD optimizations])
+ ROLL_SIMD='$(ROLL_SIMD_'"$ROLL_SIMD)"
# We only use c++ for its target attribute dispatching, disable unneeded bulky features
CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
# Apple often has "g++" as a symlink for clang. Try to find out the truth.
@@ -311,7 +311,7 @@ else
AC_MSG_RESULT(no)
fi
-AC_SUBST(SIMD)
+AC_SUBST(ROLL_SIMD)
AC_MSG_CHECKING([if assembler accepts noexecstack])
OLD_CFLAGS="$CFLAGS"
@@ -327,14 +327,14 @@ AC_SUBST(NOEXECSTACK)
AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[
AC_RUN_IFELSE([AC_LANG_SOURCE([[
#define _FILE_OFFSET_BITS 64
-#include <stdio.h>
-#include <fcntl.h>
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
+$ac_includes_default
+#ifdef HAVE_FCNTL_H
+# include <fcntl.h>
+#elif defined HAVE_SYS_FCNTL_H
+# include <sys/fcntl.h>
#endif
+#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
-#if HAVE_UNISTD_H
-#include <unistd.h>
#endif
int main(void)
@@ -433,45 +433,66 @@ if test x"$enable_openssl" != x"no"; then
err_msg="$err_msg$nl- Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support."
no_lib="$no_lib openssl"
fi
- if test x"$enable_asm" != x"yes"; then
- enable_asm=no
+ if test x"$enable_md5_asm" != x"yes"; then
+ enable_md5_asm=no
fi
else
AC_MSG_RESULT(no)
fi
-ASM=
+MD5_ASM=
-AC_MSG_CHECKING([whether to enable ASM optimizations])
-AC_ARG_ENABLE(asm,
- AS_HELP_STRING([--enable-asm],[enable/disable to control ASM optimizations]))
+AC_MSG_CHECKING([whether to enable MD5 ASM optimizations])
+AC_ARG_ENABLE(md5-asm,
+ AS_HELP_STRING([--enable-md5-asm],[enable/disable to control MD5 ASM optimizations]))
-if test x"$enable_asm" = x""; then
+if test x"$enable_md5_asm" = x""; then
case "$host_os" in
*linux*) ;;
- *) enable_asm=no ;;
+ *) enable_md5_asm=no ;;
esac
fi
-if test x"$enable_asm" != x"no"; then
+if test x"$enable_md5_asm" != x"no"; then
if test x"$host_cpu" = x"x86_64" || test x"$host_cpu" = x"amd64"; then
- ASM="$host_cpu"
- elif test x"$enable_asm" = x"yes"; then
+ MD5_ASM="$host_cpu"
+ elif test x"$enable_md5_asm" = x"yes"; then
AC_MSG_RESULT(unavailable)
AC_MSG_ERROR(The ASM optimizations are currently x86_64|amd64 only.
-Omit --enable-asm to continue without it.)
+Omit --enable-md5-asm to continue without it.)
fi
fi
-if test x"$ASM" != x""; then
- AC_MSG_RESULT([yes ($ASM)])
- AC_DEFINE(HAVE_ASM, 1, [Define to 1 to enable ASM optimizations])
- ASM='$(ASM_'"$ASM)"
+if test x"$MD5_ASM" != x""; then
+ AC_MSG_RESULT([yes ($MD5_ASM)])
+ AC_DEFINE(USE_MD5_ASM, 1, [Define to 1 to enable MD5 ASM optimizations])
+ MD5_ASM='$(MD5_ASM_'"$MD5_ASM)"
+else
+ AC_MSG_RESULT(no)
+fi
+
+AC_SUBST(MD5_ASM)
+
+ROLL_ASM=
+
+AC_MSG_CHECKING([whether to enable rolling-checksum ASM optimizations])
+AC_ARG_ENABLE(roll-asm,
+ AS_HELP_STRING([--enable-roll-asm],[enable/disable to control rolling-checksum ASM optimizations (requires --enable-roll-simd)]))
+
+if test x"$ROLL_SIMD" = x""; then
+ enable_roll_asm=no
+fi
+
+if test x"$enable_roll_asm" = x"yes"; then
+ ROLL_ASM="$host_cpu"
+ AC_MSG_RESULT([yes ($ROLL_ASM)])
+ AC_DEFINE(USE_ROLL_ASM, 1, [Define to 1 to enable rolling-checksum ASM optimizations (requires --enable-roll-simd)])
+ ROLL_ASM='$(ROLL_ASM_'"$ROLL_ASM)"
else
AC_MSG_RESULT(no)
fi
-AC_SUBST(ASM)
+AC_SUBST(ROLL_ASM)
AC_MSG_CHECKING([whether to enable xxhash checksum support])
AC_ARG_ENABLE([xxhash],
@@ -1421,10 +1442,6 @@ esac
AC_CONFIG_FILES([Makefile lib/dummy zlib/dummy popt/dummy shconfig])
AC_OUTPUT
-if test "$enable_openssl" = yes && test "$enable_asm" = yes; then
- echo "*** Ignoring --enable-asm option -- using openssl for MD5 checksums ***"
-fi
-
AC_MSG_RESULT()
AC_MSG_RESULT([ rsync $PACKAGE_VERSION configuration successful])
AC_MSG_RESULT()
diff --git a/lib/md5-asm-x86_64.S b/lib/md5-asm-x86_64.S
index 383f193a..3737058f 100644
--- a/lib/md5-asm-x86_64.S
+++ b/lib/md5-asm-x86_64.S
@@ -27,7 +27,7 @@
#include "config.h"
#include "md-defines.h"
-#if !defined USE_OPENSSL && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
#ifdef __APPLE__
#define md5_process_asm _md5_process_asm
@@ -698,4 +698,4 @@ md5_process_asm:
pop %rbp
ret
-#endif /* !USE_OPENSSL ... */
+#endif /* } USE_MD5_ASM */
diff --git a/lib/md5.c b/lib/md5.c
index 41f158b8..07fd6147 100644
--- a/lib/md5.c
+++ b/lib/md5.c
@@ -20,7 +20,7 @@
#include "rsync.h"
-#ifndef USE_OPENSSL
+#if !defined USE_OPENSSL || USE_MD5_ASM /* { */
void md5_begin(md_context *ctx)
{
ctx->A = 0x67452301;
@@ -148,7 +148,10 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK])
ctx->D += D;
}
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM
+#if CSUM_CHUNK != 64
+#error The MD5 ASM code does not support CSUM_CHUNK != 64
+#endif
extern void md5_process_asm(md_context *ctx, const void *data, size_t num);
#endif
@@ -176,20 +179,20 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length)
left = 0;
}
-#if defined HAVE_ASM && CSUM_CHUNK == 64
+#ifdef USE_MD5_ASM /* { */
if (length >= CSUM_CHUNK) {
uint32 chunks = length / CSUM_CHUNK;
md5_process_asm(ctx, input, chunks);
length -= chunks * CSUM_CHUNK;
input += chunks * CSUM_CHUNK;
}
-#else
+#else /* } { */
while (length >= CSUM_CHUNK) {
md5_process(ctx, input);
length -= CSUM_CHUNK;
input += CSUM_CHUNK;
}
-#endif
+#endif /* } */
if (length)
memcpy(ctx->buffer + left, input, length);
@@ -221,9 +224,9 @@ void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN])
SIVALu(digest, 8, ctx->C);
SIVALu(digest, 12, ctx->D);
}
-#endif
+#endif /* } */
-#ifdef TEST_MD5
+#ifdef TEST_MD5 /* { */
void get_md5(uchar *out, const uchar *input, int n)
{
@@ -317,4 +320,4 @@ int main(int argc, char *argv[])
return 0;
}
-#endif
--
The rsync repository.
More information about the rsync-cvs
mailing list