[Bug 10322] Slow Performance over Network rsync

[Linda Walsh]

samba-bugs at samba.org wrote:
> https://bugzilla.samba.org/show_bug.cgi?id=10322
>
> --- Comment #1 from roland <devzero at web.de> 2013-12-14 13:24:46 UTC ---
>   
>> rsync 3.0.8 runs with 45 mb/sec
>>     
>
> 45mb/s with cygwin?
> are you really sure?
>
> afaik, rsync via cygwin always is dead slow, especially in "local mode", which
> you are using. (as you read from a network share via unc path)
>   
----
I get 400+MB read and write over cygwin to/from remote sockets:

/h> iotest
R:512+0 records in
512+0 records out
4294967296 bytes (4.3 GB) copied, 10.5159 s, 408 MB/s
W:512+0 records in
512+0 records out
4294967296 bytes (4.3 GB) copied, 8.29549 s, 518 MB/s
/h> iotest
R:512+0 records in
512+0 records out
4294967296 bytes (4.3 GB) copied, 10.6297 s, 404 MB/s
W:512+0 records in
512+0 records out
4294967296 bytes (4.3 GB) copied, 9.42769 s, 456 MB/s

----
Any slowdowns are coming from client apps (likely doing too small of 
read/write
operations).  If you use small io buffers (4K, you can slow that 
transfer speed down
by a factor of 100.  (Mozilla is has been great at this using 4K I/O 
blocks for
local file RW and network operations). 

The above READ is reading from /dev/zero on the linux machine and writing
to /dev/null on the local windows machine.

The above WRITE is writing to a /dev/null on the remote machine and reading
from a /dev/zero on the win machine.   I created files 'null' and 'zero' 
which are
the device files '/dev/null and /dev/zero' (named null and zero) in my 
home directory
to access the devices.  I use the cygwin /dev/null & /dev/zero on the 
win side as I am
using dd.

So if someone wants to talk about "slow" windows-shares -- it's likely the
application being used.

Those messages come from 'dd'.

My test script I'll attach below.
------------------------------------------------------
#!/bin/bash -u
_prgpth="${0:?}"; _prg="${_prgpth##*/}"; _prgdr="${_prgpth%/$_prg}"
[[ -z $_prgdr || $_prg == $_prgdr ]] && $_prgdr="$PWD"
export PATH="$_prgdr:$_prgdr/lib:$PATH"
shopt -s expand_aliases extglob sourcepath ; set -o pipefail

#include stdalias

Dd=$(type -P dd)

[[ $Dd ]] || { echo "Cannot find dd.  Cannot proceed.";  exit 1; }

alias intConst=declare\ -ix int=declare\ -i my=declare
alias string=declare sub=function array=declare\ -a
# 1 num = block size
# num-num = range of block sizes to test; w/increment = "2x", so
# 4M-16M tests 4M, 8M, 16M
# 4M-12M test 4M, 8M, 12M
# count adjusted to xfer 4G, rounding up
#----

#all xfers are using 'devices' (/dev/zero for source, /dev/null for target)
# remote filenames "zero" and "null" should be setup to be remote devices

intConst K=1024
intConst M=$[K*K]
intConst G=$[M*K]
intConst T=$[G*K]

int BS=$[8*M]
int count=512
int IOSIZE=${IOSIZE:-4*G}

#        desuffix     1st arg = num+suffix -> convert to int
#                            2nd arg = optional buff name (else print to 
stdout)
#                            return 0 if no error

sub desuffix {                        #convert num+Suff => int store in 
optional Buff
    string str="${1:?}" ; shift
    string bufnam=""; (($#)) && bufnam=$1
    if [[ $p =~ ^([0-9]+)([KMGT])$ ]]; then
        int num=${BASH_REMATCH[1]}*${BASH_REMATCH[2]}
        ((num)) || return 1
        if [[ $bufnam ]] ; then printf -v $bufnam "%d" "$num"
        else printf "%d" "$num" ; fi
    else
        return 1
    fi
}




sub hdisp {
    int num=${1:?}; shift
    string bufnam=""; (($#)) && bufnam=$1
    string suf=""
    int ans=num
    array pows=('K' 'M' 'G' 'T')
    for s in ${pows[@]};do
        int si=$s
        if (((num/si)*si==num)); then
            ans=num/si;suf="$s"
        fi
    done
}

sub check_params {
    int num=0
    if (($#)) ; then
        string p="$1"; shift;
        if [[ $p =~ ([0-9]+)([KMGT]) ]]; then
            num=${BASH_REMATCH[1]}*${BASH_REMATCH[2]}
        fi
    fi
    ((num)) && {
        BS=num
        count=IOSIZE/BS
    }
}

(($#)) && check_params "$@"

array reada=(/h/zero /dev/null)
array writea=(/dev/zero /h/null conv=nocreat,notrunc)

sub dd_need_io  {
    local if="$1" of="$2"; shift 2
    nice --19 $Dd if="$if" of="$of" bs="$BS" count="$count"\
             oflag=direct iflag=direct conv=nocreat "$@"
}

sub dd {
    local if="$1" of="$2" ;shift 2
    #echo $dd if="$if" of="$of" bs="$BS" count="$count" "$@" >&2
    array out err
    readarray err < <( \
        readarray out < <(dd_need_io "$if" "$of" "$@";int s=$?;
                                            ((s)) && echo "stat:$s">&2  
) 2>&1 )
    #
    if ((${#err[@]})) ;then echo "${err[@]}"; exit 1; fi
    return 0
}
   
function dd_format { my ln
    while read ln;do
        echo $ln | while read bytes btxt pnum suffp \
                                            copt time unitc rate ra_unit; do
            [[ $bytes == records ]] && continue
            [[ ${pnum:0:1} != \( || ${suffp:0-1:1} != \) ]] && continue
            num="${pnum:1}"
            suff="${suffp%?}"
            unit="${unitc%?}"
            printf "%s:%s:%s:%s:%s:%s:\n"    "$num" "$suff" "$time" 
"$unit" "$rate" "$ra_unit"
        done
    done
}

sub onecycle {
    echo -n "R:"; { dd "${reada[@]}"|| exit $?; } | dd_format
    echo -n "W:";    { dd "${writea[@]}" || exit $?; } | dd_format
}

onecycle