[Samba] smbd's using up 100% of all cpu's and load avg slowly going up

Mon Dec 29 16:46:18 MST 2014

Hi,

I update to sernet-samba 4.1.14, everything started ok, but after I connect the first client and working with files in shares, the process returned with a CPU usage between 90% and 100%. After I logoff the client, the process continues.

I did a 'samba-tool processes' and the process responsible is winbind_server (after pid comparison)

Now, the strace -ff -p have a little difference, some brk() syscalls:
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=29212, len=1}) = 0
brk(0x7f99e70d9000)                     = 0x7f99e70d9000
writev(44, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}], 1) = 3496
epoll_wait(9, {{EPOLLIN, {u32=3862499424, u64=140298969223264}}}, 1, 30000) = 1
readv(34, [{"0\10\0\0", 4}], 1)         = 4
epoll_wait(9, {{EPOLLIN, {u32=3853966256, u64=140298960690096}}}, 1, 30000) = 1
readv(44, [{"0\10\0\0", 4}], 1)         = 4
epoll_wait(9, {{EPOLLIN, {u32=3862499424, u64=140298969223264}}}, 1, 30000) = 1
readv(34, [{"\35\0\0\0\0\0\0\0\362\n\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2092}], 1) = 2092
gettimeofday({1419895514, 796889}, NULL) = 0
gettimeofday({1419895514, 796954}, NULL) = 0
gettimeofday({1419895514, 797014}, NULL) = 0
gettimeofday({1419895514, 797082}, NULL) = 0
gettimeofday({1419895514, 797155}, NULL) = 0
gettimeofday({1419895514, 797229}, NULL) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=39416, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=39416, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=11800, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=11800, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=15836, len=1}) = 0
brk(0x7f99e712b000)                     = 0x7f99e712b000
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=15836, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=29212, len=1}) = 0
brk(0x7f99e7179000)                     = 0x7f99e7179000
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=29212, len=1}) = 0
brk(0x7f99e70d9000)                     = 0x7f99e70d9000
writev(34, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}], 1) = 3496
epoll_wait(9, {{EPOLLIN, {u32=3853966256, u64=140298960690096}}}, 1, 30000) = 1
readv(44, [{"\34\0\0\0\0\0\0\0\361\f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2092}], 1) = 2092
gettimeofday({1419895514, 817239}, NULL) = 0
gettimeofday({1419895514, 817877}, NULL) = 0
gettimeofday({1419895514, 817940}, NULL) = 0
gettimeofday({1419895514, 818007}, NULL) = 0
gettimeofday({1419895514, 818141}, NULL) = 0
gettimeofday({1419895514, 818218}, NULL) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14576, len=1}) = 0
brk(0x7f99e7130000)                     = 0x7f99e7130000
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14576, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=17520, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=17520, len=1}) = 0
brk(0x7f99e7189000)                     = 0x7f99e7189000
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=16112, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=16112, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=252, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=252, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=13616, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=13616, len=1}) = 0
brk(0x7f99e70d9000)                     = 0x7f99e70d9000
writev(44, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}], 1) = 3496
epoll_wait(9, {{EPOLLIN, {u32=3862499424, u64=140298969223264}}}, 1, 30000) = 1
readv(34, [{"0\10\0\0", 4}], 1)         = 4
epoll_wait(9, {{EPOLLIN, {u32=3853966256, u64=140298960690096}}}, 1, 30000) = 1
readv(44, [{"0\10\0\0", 4}], 1)         = 4
epoll_wait(9, {{EPOLLIN, {u32=3862499424, u64=140298969223264}}}, 1, 30000) = 1
readv(34, [{"\34\0\0\0\0\0\0\0\362\n\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2092}], 1) = 2092
gettimeofday({1419895514, 824981}, NULL) = 0
gettimeofday({1419895514, 825045}, NULL) = 0
gettimeofday({1419895514, 825103}, NULL) = 0
gettimeofday({1419895514, 825168}, NULL) = 0
gettimeofday({1419895514, 825237}, NULL) = 0
gettimeofday({1419895514, 825454}, NULL) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14576, len=1}) = 0
brk(0x7f99e7130000)                     = 0x7f99e7130000
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14576, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=17520, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=17520, len=1}) = 0
brk(0x7f99e7189000)                     = 0x7f99e7189000
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=16112, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=16112, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=1040, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=1040, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=13464, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=13464, len=1}) = 0
brk(0x7f99e70d9000)                     = 0x7f99e70d9000
writev(34, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}], 1) = 3496
epoll_wait(9, {{EPOLLIN, {u32=3853966256, u64=140298960690096}}}, 1, 30000) = 1
readv(44, [{"\35\0\0\0\0\0\0\0\361\f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 2092}], 1) = 2092
gettimeofday({1419895514, 831495}, NULL) = 0
gettimeofday({1419895514, 831559}, NULL) = 0
gettimeofday({1419895514, 831617}, NULL) = 0
gettimeofday({1419895514, 831687}, NULL) = 0
gettimeofday({1419895514, 831758}, NULL) = 0
gettimeofday({1419895514, 831833}, NULL) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=39416, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=39416, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=11800, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=11800, len=1}) = 0
fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=15836, len=1}) = 0
brk(0x7f99e712b000)                     = 0x7f99e712b000

fd - 13 ->  /var/lib/samba/private/sam.ldb.d/DC=EXAMPLE,DC=COM.ldb
fd - 16 ->  /var/lib/samba/private/sam.ldb.d/metadata.tdb

Regards,
Bruno Andrade.

Em Segunda, Dezembro de 29 de 2014 15:14 WET, Rowland Penny <rowlandpenny at googlemail.com> escreveu:

> On 29/12/14 14:58, Bruno Andrade wrote:
> > Hi,
> >
> > Yes, I already thought about it.
> >
> > I have the sernet repo installed.. Is possible to yum update and jump
> > from 4.1.6 to 4.1.14 directly, without problems or should I follow some
> > kind of procedure to avoid future problems?
> >
> > Regards,
> > Bruno Andrade.
> >
> > On 12/29/2014 01:46 PM, Rowland Penny wrote:
> >> On 29/12/14 12:02, Bruno Andrade wrote:
> >>> Hi again,
> >>>
> >>> Sorry, I forgot to include the fd used...
> >>>
> >>> 13 -> /var/lib/samba/private/sam.ldb.d/DC=EXAMPLE,DC=COM.ldb
> >>> 16 -> /var/lib/samba/private/sam.ldb.d/metadata.tdb
> >>>
> >>> the others refers to /var/run/samba/winbindd/pipe
> >>>
> >>> Regards,
> >>> Bruno Andrade.
> >>>
> >>> On 12/29/2014 11:56 AM, Bruno Andrade wrote:
> >>>> Hi,
> >>>>
> >>>> Here is some output of "strace -ff -p" to the process using ~100% CPU
> >>>>
> >>>> writev(75, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}],
> >>>> 1) = 3496
> >>>> epoll_wait(9, {{EPOLLIN, {u32=2539833808, u64=140271876753872}}}, 1,
> >>>> 30000) = 1
> >>>> readv(91,
> >>>> [{"\34\0\0\0\0\0\0\0rT\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"...,
> >>>> 2092}],
> >>>> 1) = 2092
> >>>> gettimeofday({1419854003, 222035}, NULL) = 0
> >>>> gettimeofday({1419854003, 222103}, NULL) = 0
> >>>> gettimeofday({1419854003, 222169}, NULL) = 0
> >>>> fcntl(16, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=176,
> >>>> len=1}) = 0
> >>>> fcntl(16, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=176,
> >>>> len=1}) = 0
> >>>> gettimeofday({1419854003, 222345}, NULL) = 0
> >>>> gettimeofday({1419854003, 222482}, NULL) = 0
> >>>> gettimeofday({1419854003, 222868}, NULL) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14576,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14576,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=17520,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=17520,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=16112,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=16112,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=1488,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=1488,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=15644,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=15644,
> >>>> len=1}) = 0
> >>>> writev(91, [{"\250\r\0\0\2\0\0\0S-1-5-21-2097307442-3435"..., 3496}],
> >>>> 1) = 3496
> >>>> epoll_wait(9, {{EPOLLIN, {u32=2540085760, u64=140271877005824}}}, 1,
> >>>> 30000) = 1
> >>>> readv(41,
> >>>> [{"\34\0\0\0\0\0\0\0WS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"...,
> >>>> 2092}],
> >>>> 1) = 2092
> >>>> gettimeofday({1419854003, 230871}, NULL) = 0
> >>>> gettimeofday({1419854003, 230939}, NULL) = 0
> >>>> gettimeofday({1419854003, 231002}, NULL) = 0
> >>>> fcntl(16, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=176,
> >>>> len=1}) = 0
> >>>> fcntl(16, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=176,
> >>>> len=1}) = 0
> >>>> gettimeofday({1419854003, 231249}, NULL) = 0
> >>>> gettimeofday({1419854003, 231328}, NULL) = 0
> >>>> gettimeofday({1419854003, 231474}, NULL) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14324,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14324,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_RDLCK, whence=SEEK_SET, start=14576,
> >>>> len=1}) = 0
> >>>> fcntl(13, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=14576,
> >>>> len=1}) = 0