rsync freezes on solaris

[Anand Raman]

hi guys
 
We use rsync extensively in keeping the site resources in synch across redundant infrastructure.
While attempting rsync for one of the boxes we find that the rsync process simply sleeps. The situation remains the same even if we have sync very small contents of the order of 4 KB. The same run with the -an flag runs without any issues. We run rsync over ssh on solaris boxes.
To debug the situation we attached "truss" to the process on the source as well as the destination. 
 
We see the following output on the destination server. 
<snip>
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 0
poll(0xFFBEF4B8, 2, 60000) (sleeping...)
poll(0xFFBEF4B8, 2, 60000)   = 1
poll(0xFFBEF218, 1, 60000)   = 1
read(7, " ?\0\0\b", 4)    = 4
poll(0xFFBEF218, 1, 60000)   = 1
read(7, " r s y n c :   c o n n e".., 63) = 63
poll(0xFFBEE0E8, 1, 60000)   = 1
write(1, " ?\0\0\b r s y n c :   c".., 67) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
write(2, " r s y n c :   w r i t e".., 81) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEDAF8, 0xFFBEDB78) = 0
sigaction(SIGUSR2, 0xFFBEDAF8, 0xFFBEDB78) = 0
getpid()     = 7970 [7969]
    Received signal #18, SIGCLD [caught]
      siginfo: SIGCLD CLD_EXITED pid=7971 status=0x000C
waitid(P_ALL, 0, 0xFFBED6D8, WEXITED|WTRAPPED|WNOHANG) = 0
waitid(P_ALL, 0, 0xFFBED6D8, WEXITED|WTRAPPED|WNOHANG) Err#10 ECHILD
setcontext(0xFFBED890)
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBED598, 0xFFBED618) = 0
sigaction(SIGUSR2, 0xFFBED598, 0xFFBED618) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBED038, 0xFFBED0B8) = 0
sigaction(SIGUSR2, 0xFFBED038, 0xFFBED0B8) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBECAD8, 0xFFBECB58) = 0
sigaction(SIGUSR2, 0xFFBECAD8, 0xFFBECB58) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEC578, 0xFFBEC5F8) = 0
sigaction(SIGUSR2, 0xFFBEC578, 0xFFBEC5F8) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEC018, 0xFFBEC098) = 0
sigaction(SIGUSR2, 0xFFBEC018, 0xFFBEC098) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEBAB8, 0xFFBEBB38) = 0
sigaction(SIGUSR2, 0xFFBEBAB8, 0xFFBEBB38) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEB558, 0xFFBEB5D8) = 0
sigaction(SIGUSR2, 0xFFBEB558, 0xFFBEB5D8) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEAFF8, 0xFFBEB078) = 0
sigaction(SIGUSR2, 0xFFBEAFF8, 0xFFBEB078) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEAA98, 0xFFBEAB18) = 0
sigaction(SIGUSR2, 0xFFBEAA98, 0xFFBEAB18) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
sigaction(SIGUSR1, 0xFFBEA538, 0xFFBEA5B8) = 0
sigaction(SIGUSR2, 0xFFBEA538, 0xFFBEA5B8) = 0
getpid()     = 7970 [7969]
kill(7971, SIGUSR1)    Err#3 ESRCH
write(2, " r s y n c   e r r o r :".., 69) Err#32 EPIPE
    Received signal #13, SIGPIPE [ignored]
llseek(0, 0, SEEK_CUR)    Err#9 EBADF
_exit(13)
</snip>
 
The rsync process also terminates unexpectedly on the remote server and I suspect thats what is causing the freeze to happen.
 
Any help in figuring out this issue will be greatly appreciated. Please let me know if I have missed some information which would have further helped in diagnosing the issue
 
Thanks for your time
anand