[Samba] CTDB + Samba: Tune Read Performance

[tim clusters]

Hi,

I have a two server setup that acts as SMB as well as NFS servers in
active/active configuration managed by CTDB(http://ctdb.samba.org/).

The write performance is around 100MB/s per client however the read
performance is only 0.6MB/s (using Iozone benchmark). I use Windows 2003
Server as CIFS client. Sometimes the read performance is good only from one
of the CTDB managed Samba servers but not consistent when you restart CTDB +
Samba.

Raw network bandwidth between CIFS client and Samba is greater than 200MB/s.
I tried changing the SO_RCVBUF and SO_SNDBUF size and smb.conf locking
options. Tracing smbd indicates sendfile being used for read and no errors
(-EINVAL or -EIO).

Any idea what is contributing the overhead during the read operation?
smb.conf is as follows:

 [global]
        workgroup = TESTDOMAIN2
        realm = TESTDOMAIN2.LOCAL
        netbios name = CTDB-HEAD
        server string = Clustered CIFS
        security = ADS
        auth methods = winbind, sam
        password server = 172.16.2.25
        private dir = /mnt/gpfs/CTDB_AD
        passdb backend = tdbsam
        log level = 1
        log file = /var/log/samba/log.%m
        max log size = 10000
        deadtime = 15
        socket options = IPTOS_LOWDELAY TCP_NODELAY SO_RCVBUF=262144
SO_SNDBUF=262144 SO_KEEPALIVE
        use mmap = No
        clustering = Yes
        disable spoolss = Yes
        machine password timeout = 999999999
        local master = No
        dns proxy = No
        ldap admin dn = cn=ldap,cn=Users,dc=testdomain2,dc=local
        ldap idmap suffix = dc=testdomain2,dc=local
        ldap suffix = dc=testdomain2,dc=local
        idmap backend = ad
        idmap uid = 5000-100000000
        idmap gid = 5000-100000000
        template homedir = /home/%D+%U
        template shell = /bin/bash
        winbind separator = +
        winbind enum users = Yes
        winbind enum groups = Yes
        notify:inotify = no
        idmap:cache = no
        gpfs:leases = yes
        nfs4:acedup = merge
        nfs4:chown = yes
        nfs4:mode = special
        gpfs:sharemodes = yes
        fileid:mapping = fsname
        force unknown acl user = Yes
        use sendfile = Yes
        mangled names = No
        blocking locks = No
        strict locking = No
        wide links = No
        vfs objects = syncops, gpfs, fileid
        large readwrite = yes
        oplocks = yes
        getwd cache = yes
        blocking locks = no
[global-share]
        comment = GPFS File Share
        path = /mnt/gpfs/nfsexport
        read only = No
        inherit permissions = Yes
        inherit acls = Yes
        oplocks = yes
Thanks in Advance,
-Tim