PATCH: --write-devices to allow synchronising to a block device

Darryl Dixon - Winterhouse Consulting darryl.dixon at winterhouseconsulting.com
Thu Oct 15 03:05:34 MDT 2009


Hi List,

I had a need recently to efficiently synchronise between some large LUNs
(boot drive disks) at two different datacentres. Solutions like drbd and
$proprietary_array_vendors_software were overkill - we only needed
(wanted!) to periodically synchronise these LUNs whenever major changes
were generated on the source. On the other hand however, re-sending the
entire disk contents each time would have been prohibitive.

So, I immediately thought about rsync. However, I discovered two problems:
1) The default build doesn't want to read from block device files
2) The default build doesn't want to write to block device files

It turned out that (1) was easy to solve as there is a --copy-devices
patch in the rsync-patches distribution that delivers this functionality
(read from block device files). Seems that nobody before me however had
needed/wanted to be able to do (2).

So I wrote a patch (--write-devices) which fulfills (2). The example usage
scenario for synchronising one disk to another like this would be: $ rsync
--copy-devices --write-devices /dev/sda /dev/sdb

I want to stress that, obviously --write-devices implies --inplace, and I
am exceptionally grateful to both the rsync developers and the authors of
the original --inplace code for making this possible. Additionally, I used
the --copy-devices patch for clues and some of the device sizing code -
thanks!

I have included the patch at the bottom of this mail. I would appreciate
any constructive critique etc to improve the robustness and quality of the
patch.

regards,
Darryl Dixon
Winterhouse Consulting Ltd
http://www.winterhouseconsulting.com

--------------------------------8<-------------------------------[snip]
diff -ru rsync-3.0.6/generator.c rsync-3.0.6-writedev/generator.c
--- rsync-3.0.6/generator.c     2009-04-27 02:51:50.000000000 +1200
+++ rsync-3.0.6-writedev/generator.c    2009-10-15 20:54:07.000000000 +1300
@@ -39,6 +39,7 @@
 extern int preserve_xattrs;
 extern int preserve_links;
 extern int preserve_devices;
+extern int write_devices;
 extern int preserve_specials;
 extern int preserve_hard_links;
 extern int preserve_executability;
@@ -1733,7 +1734,7 @@
        fnamecmp = fname;
        fnamecmp_type = FNAMECMP_FNAME;

-       if (statret == 0 && !S_ISREG(sx.st.st_mode)) {
+       if (statret == 0 && !(S_ISREG(sx.st.st_mode) || (write_devices &&
IS_DEVICE(sx.st.st_mode)))) {
                if (delete_item(fname, sx.st.st_mode, del_opts |
DEL_FOR_FILE) != 0)
                        goto cleanup;
                statret = -1;
diff -ru rsync-3.0.6/options.c rsync-3.0.6-writedev/options.c
--- rsync-3.0.6/options.c       2009-04-13 08:01:14.000000000 +1200
+++ rsync-3.0.6-writedev/options.c      2009-10-15 20:56:18.000000000 +1300
@@ -48,6 +48,7 @@
 int keep_dirlinks = 0;
 int copy_dirlinks = 0;
 int copy_links = 0;
+int write_devices = 0;
 int preserve_links = 0;
 int preserve_hard_links = 0;
 int preserve_acls = 0;
@@ -350,6 +351,7 @@
   rprintf(F," -o, --owner                 preserve owner (super-user
only)\n");
   rprintf(F," -g, --group                 preserve group\n");
   rprintf(F,"     --devices               preserve device files
(super-user only)\n");
+  rprintf(F," -w  --write-devices         write to devices as regular
files (implies --inplace)\n");
   rprintf(F,"     --specials              preserve special files\n");
   rprintf(F," -D                          same as --devices --specials\n");
   rprintf(F," -t, --times                 preserve modification times\n");
@@ -508,6 +510,7 @@
   {"no-D",             0,  POPT_ARG_NONE,   0, OPT_NO_D, 0, 0 },
   {"devices",          0,  POPT_ARG_VAL,    &preserve_devices, 1, 0, 0 },
   {"no-devices",       0,  POPT_ARG_VAL,    &preserve_devices, 0, 0, 0 },
+  {"write-devices",   'w', POPT_ARG_NONE,   0, 'w', 0, 0 },
   {"specials",         0,  POPT_ARG_VAL,    &preserve_specials, 1, 0, 0 },
   {"no-specials",      0,  POPT_ARG_VAL,    &preserve_specials, 0, 0, 0 },
   {"links",           'l', POPT_ARG_VAL,    &preserve_links, 1, 0, 0 },
@@ -1261,6 +1264,11 @@
                        return 0;
 #endif

+                case 'w':
+                        write_devices = 1;
+                        inplace = 1;
+                        break;
+
                default:
                        /* A large opt value means that set_refuse_options()
                         * turned this option off. */
@@ -2069,6 +2077,9 @@
        else if (remove_source_files)
                args[ac++] = "--remove-sent-files";

+        if (write_devices)
+                args[ac++] = "--write-devices";
+
        if (ac > MAX_SERVER_ARGS) { /* Not possible... */
                rprintf(FERROR, "argc overflow in server_options().\n");
                exit_cleanup(RERR_MALLOC);
diff -ru rsync-3.0.6/receiver.c rsync-3.0.6-writedev/receiver.c
--- rsync-3.0.6/receiver.c      2009-04-13 07:48:59.000000000 +1200
+++ rsync-3.0.6-writedev/receiver.c     2009-10-15 20:54:22.000000000 +1300
@@ -38,6 +38,7 @@
 extern int relative_paths;
 extern int preserve_hard_links;
 extern int preserve_perms;
+extern int write_devices;
 extern int preserve_xattrs;
 extern int basis_dir_cnt;
 extern int make_backups;
@@ -165,6 +166,7 @@
 static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
                        const char *fname, int fd, OFF_T total_size)
 {
+        STRUCT_STAT st;
        static char file_sum1[MAX_DIGEST_LEN];
        static char file_sum2[MAX_DIGEST_LEN];
        struct map_struct *mapbuf;
@@ -285,10 +287,14 @@
                goto report_write_error;

 #ifdef HAVE_FTRUNCATE
-       if (inplace && fd != -1
-        && ftruncate(fd, offset) < 0) {
-               rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
-                       full_fname(fname));
+        (void)do_fstat(fd,&st);
+        /* Makes no sense to attempt to ftruncate() a block device: */
+        if (!(IS_DEVICE(st.st_mode))) {
+               if (inplace && fd != -1
+                && ftruncate(fd, offset) < 0) {
+                       rsyserr(FERROR_XFER, errno, "ftruncate failed on %s",
+                               full_fname(fname));
+                }
        }
 #endif

@@ -668,11 +674,25 @@
                        continue;
                }

-               if (fd1 != -1 && !S_ISREG(st.st_mode)) {
+               if (fd1 != -1 && !(S_ISREG(st.st_mode) || (write_devices
&& IS_DEVICE(st.st_mode)))) {
                        close(fd1);
                        fd1 = -1;
                }

+                /* On Linux systems (at least), st_size is typically 0
for devices.
+                 * If so, try to determine the actual device size. */
+                if (fd1 != -1 && IS_DEVICE(st.st_mode) && st.st_size == 0) {
+                        OFF_T off = lseek(fd1, 0, SEEK_END);
+                        if (off == (OFF_T) -1)
+                                rsyserr(FERROR, errno, "failed to seek to
end of %s to determine size", fname);
+                        else {
+                                st.st_size = off;
+                                off = lseek(fd1, 0, SEEK_SET);
+                                if (off != 0)
+                                        rsyserr(FERROR, errno, "failed to
seek back to beginning of %s to read it", fname);
+                        }
+                }
+
                /* If we're not preserving permissions, change the
file-list's
                 * mode based on the local permissions and some
heuristics. */
                if (!preserve_perms) {

--------------------------------8<-------------------------------[snip]


More information about the rsync mailing list