[PATCH] Prefer fallocate if available, and use posix_fallocate only on Cygwin.

Matt McCutchen matt at mattmccutchen.net
Sun Feb 24 03:49:25 GMT 2008


---
This change on top of the current preallocate branch implements the
behavior I described of using fallocate if available (Linux syscall that
uses filesystem-level preallocation support) or otherwise
posix_fallocate if on Cygwin.  This patch works on my computer (i.e., it
makes the fallocate syscall), but it could use additional testing,
especially on Cygwin.

Matt

 configure.in |   25 ++++++++++++++++++++++++-
 options.c    |    2 +-
 receiver.c   |    4 ++--
 rsync.h      |    3 ++-
 rsync.yo     |   15 +++++++--------
 syscall.c    |   20 ++++++++++++++++++++
 util.c       |    6 +++---
 7 files changed, 59 insertions(+), 16 deletions(-)

diff --git a/configure.in b/configure.in
index c8e9394..36c9007 100644
--- a/configure.in
+++ b/configure.in
@@ -552,13 +552,36 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \
     strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \
     setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
     strerror putenv iconv_open locale_charset nl_langinfo getxattr \
-    extattr_get_link sigaction sigprocmask setattrlist posix_fallocate)
+    extattr_get_link sigaction sigprocmask setattrlist fallocate posix_fallocate)
 
 dnl cygwin iconv.h defines iconv_open as libiconv_open
 if test x"$ac_cv_func_iconv_open" != x"yes"; then
     AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)])
 fi
 
+dnl Preallocation stuff (also fallocate, posix_fallocate function tests above):
+
+AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[
+AC_TRY_COMPILE([#include <sys/syscall.h>
+#include <sys/types.h>],
+[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);],
+rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)])
+if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then
+    AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number])
+fi
+
+dnl Copied from SHUTDOWN_ALL_SOCKETS test
+AC_MSG_CHECKING([whether posix_fallocate is efficient])
+case $host_os in
+	*cygwin* ) AC_MSG_RESULT(yes)
+                   AC_DEFINE(POSIX_FALLOCATE_IS_EFFICIENT, 1,
+			    [Define if posix_fallocate is efficient (Cygwin)])
+		   ;;
+	       * ) AC_MSG_RESULT(no);;
+esac
+
+dnl End of preallocation stuff
+
 AC_CHECK_FUNCS(getpgrp tcgetpgrp)
 if test $ac_cv_func_getpgrp = yes; then
     AC_FUNC_GETPGRP
diff --git a/options.c b/options.c
index 460966c..478e315 100644
--- a/options.c
+++ b/options.c
@@ -363,7 +363,7 @@ void usage(enum logcode F)
 #endif
   rprintf(F," -S, --sparse                handle sparse files efficiently\n");
 #ifdef SUPPORT_PREALLOCATION
-  rprintf(F,"     --preallocate           posix_fallocate dest files before writing them\n");
+  rprintf(F,"     --preallocate           allocate dest files before writing them\n");
 #endif
   rprintf(F," -n, --dry-run               perform a trial run with no changes made\n");
   rprintf(F," -W, --whole-file            copy files whole (without delta-xfer algorithm)\n");
diff --git a/receiver.c b/receiver.c
index 96c3c5e..a62ece0 100644
--- a/receiver.c
+++ b/receiver.c
@@ -182,10 +182,10 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
 	if (preallocate_files && fd != -1 && total_size > 0) {
 		/* Preallocate enough space for file's eventual length if
 		 * possible; seems to reduce fragmentation on Windows. */
-		if (posix_fallocate(fd, 0, total_size) == 0)
+		if (do_fallocate(fd, 0, total_size) == 0)
 			preallocated_len = total_size;
 		else
-			rsyserr(FINFO, errno, "preallocate %s", full_fname(fname));
+			rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname));
 	}
 #endif
 
diff --git a/rsync.h b/rsync.h
index a5489f2..aeed22d 100644
--- a/rsync.h
+++ b/rsync.h
@@ -602,7 +602,8 @@ struct ht_int64_node {
 #define ACLS_NEED_MASK 1
 #endif
 
-#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE
+#if HAVE_FTRUNCATE && (HAVE_FALLOCATE || HAVE_SYS_FALLOCATE \
+	|| (HAVE_POSIX_FALLOCATE && POSIX_FALLOCATE_IS_EFFICIENT))
 #define SUPPORT_PREALLOCATION 1
 #endif
 
diff --git a/rsync.yo b/rsync.yo
index b2ff1ce..ce0566f 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -352,7 +352,7 @@ to the detailed description below for a complete description.  verb(
      --super                 receiver attempts super-user activities
      --fake-super            store/recover privileged attrs using xattrs
  -S, --sparse                handle sparse files efficiently
-     --preallocate           posix_fallocate dest files before writing
+     --preallocate           allocate dest files before writing
  -n, --dry-run               perform a trial run with no changes made
  -W, --whole-file            copy files whole (w/o delta-xfer algorithm)
  -x, --one-file-system       don't cross filesystem boundaries
@@ -1038,17 +1038,16 @@ filesystem. It doesn't seem to handle seeks over null regions
 correctly and ends up corrupting the files.
 
 dit(bf(--preallocate)) This tells the receiver to allocate each destination
-file to its eventual size using bf(posix_fallocate)(3) before writing data
-to the file.  If the receiver is remote, this nonstandard option only works
-if the receiver also has the preallocation patch.  Furthermore, this option
-only works if the receiver found the bf(posix_fallocate)(3) call at
-configure time.
+file to its eventual size before writing data to the file.  Rsync will only use
+the real filesystem-level preallocation support provided by bf(fallocate)(2) or
+Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes
+a zero byte into each block.  If the receiver is remote, this nonstandard
+option only works if the receiver also has the preallocation patch.
 
 Without this option on MS Windows, very large destination files tend to be
 broken into thousands of fragments; advising Windows ahead of time of the
 eventual file size using this option usually reduces the number of
-fragments to one.  However, on Linux, this option appears to just waste
-disk I/O.
+fragments to one.  The usefulness of this option on Linux is yet to be tested.
 
 dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't
 make any changes (and produces mostly the same output as a real run).  It
diff --git a/syscall.c b/syscall.c
index 39554e9..36f10b4 100644
--- a/syscall.c
+++ b/syscall.c
@@ -29,6 +29,10 @@
 #include <sys/attr.h>
 #endif
 
+#if !HAVE_FALLOCATE && HAVE_SYS_FALLOCATE
+#include <sys/syscall.h>
+#endif
+
 extern int dry_run;
 extern int am_root;
 extern int read_only;
@@ -291,3 +295,19 @@ char *d_name(struct dirent *di)
 	return di->d_name;
 #endif
 }
+
+#ifdef SUPPORT_PREALLOCATION
+int do_fallocate(int fd, OFF_T offset, OFF_T length)
+{
+	RETURN_ERROR_IF(dry_run, 0);
+	RETURN_ERROR_IF_RO_OR_LO;
+	/* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */
+#if HAVE_FALLOCATE
+	return fallocate(fd, 0, offset, length);
+#elif HAVE_SYS_FALLOCATE
+	return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length);
+#else /* HAVE_POSIX_FALLOCATE && POSIX_FALLOCATE_IS_EFFICIENT */
+	return posix_fallocate(fd, offset, length);
+#endif
+}
+#endif
diff --git a/util.c b/util.c
index 2603e76..43c984f 100644
--- a/util.c
+++ b/util.c
@@ -306,13 +306,13 @@ int copy_file(const char *source, const char *dest, int ofd,
 		STRUCT_STAT srcst;
 		if (do_fstat(ifd, &srcst) == 0) {
 			if (srcst.st_size > 0) {
-				if (posix_fallocate(ofd, 0, srcst.st_size) == 0)
+				if (do_fallocate(ofd, 0, srcst.st_size) == 0)
 					preallocated_len = srcst.st_size;
 				else
-					rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest));
+					rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest));
 			}
 		} else
-			rsyserr(FINFO, errno, "fstat %s", full_fname(source));
+			rsyserr(FWARNING, errno, "fstat %s", full_fname(source));
 	}
 #endif
 
-- 
1.5.4.3.193.g6dd0e



More information about the rsync mailing list