samba_spnupdate invoked oom-killer ( samba BUG )
Hemanth Thummala
hemanth.thummala at gmail.com
Mon Sep 29 00:23:50 MDT 2014
Hi Chan,
I suspect that this could be due to huge tdb(session/locking/messages) file
issue. We have seen the same case where samba process grows to this big
when it maps to tdb virtual address space. One of these tdb files were
having huge dead records. Please check if you hit the same problem.
Thanks,
Hemanth.
On Tue, Sep 23, 2014 at 4:45 PM, Chan Min Wai <dcmwai at gmail.com> wrote:
> Hello All,
>
> I've post this on the samba mail listing for a while and agreed that it was
> only developer which can help... and thus post on this list.
>
> I've a dmesg output on the last part of the mail below.
>
> I've currently increase my Vm memory to 8G but I think the problem will
> happen again.
> as there is a ps of samba is running on high memory now (only one and it is
> building up overtime)
>
> So i think we have a bugs somewhere... (memory leak or something?)
>
>
> Where should we start?
> DC2 are still on 9xx only
>
> total used free shared buffers cached
> Mem: 3.9G 2.7G 1.1G 1.2M 4.4M 781M
> -/+ buffers/cache: 2.0G 1.9G
> Swap: 1.9G 0B 1.9G
>
>
> However PS seem to show something strange
> a samba process with 1.6G memory. PID 4448
>
> I left 3 process for reference, I cannot see the different... in command
> they run
>
> Please advise what I can do to help.
>
> I'm on gentoo and this DC setup are just an classical upgrade from PDC.
>
> /etc/samba/smb.conf
> # Global parameters
> [global]
> workgroup = WORKGROUP
> realm = domain
> netbios name = DC1
> server role = active directory domain controller
> server services = s3fs, rpc, nbt, wrepl, ldap, cldap, kdc, drepl,
> winbind, ntp_signd, kcc, dnsupdate
> idmap_ldb:use rfc2307 = yes
> host msdfs = yes
>
> # DNS Update
> allow dns updates = nonsecure
>
> #Replicate KCC Connection Object Fis
> kccsrv:samba_kcc = false
>
> # Log Level
> LOG Level = 2
> max log size = 2000
>
> # Disable Load printer
> load printers = No
>
> #SMB2 Enable
> client max protocol = SMB2
>
> [netlogon]
> path = /var/lib/samba/sysvol/domain/scripts
> read only = No
>
> [sysvol]
> path = /var/lib/samba/sysvol
> read only = No
> [dfs]
> path = /home/dfsroot
> msdfs root = yes
>
>
>
> ps output
>
> root 4447 0.0 0.9 550556 37216 ? S Sep20 0:05 \_
> /usr/sbin/samba
>
> MANPATH=/usr/local/share/man:/usr/share/man:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/man:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/man
> CONSOLE=/dev/console SHELL=/bin/sh TERM=linux EINFO_LOG=/etc/init.d/samba
> MULTIOSDIRS=../lib64:../lib32 CONFIG_PROTECT_MASK=/etc/gentoo-release
> /etc/sandbox.d /etc/fonts/fonts.conf /etc/terminfo
> /etc/ca-certificates.conf /etc/revdep-rebuild RC_SVCNAME=samba PWD=/
> DISTCC_LOG= LANG=en_US.utf8 DISTCC_VERBOSE= DCCC_PATH=/usr/lib64/distcc/bin
> SVCNAME=samba SHLVL=1 HOME=/ RC_SERVICE=/etc/init.d/samba LESS=-R -M
> --shift 5 GCC_SPECS=
> ROOTPATH=/opt/bin:/usr/x86_64-pc-linux-gnu/gcc-bin/4.7.3 LESSOPEN=|lesspipe
> %s
>
> INFOPATH=/usr/share/info:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/info:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/info
> CONFIG_PROTECT=/var/bind _=/sbin/start-stop-daemon USER=root
>
> PATH=/bin:/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
>
> root 4448 4.1 42.5 2244104 1725108 ? S Sep20 124:49 \_
> /usr/sbin/samba
>
> MANPATH=/usr/local/share/man:/usr/share/man:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/man:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/man
> CONSOLE=/dev/console SHELL=/bin/sh TERM=linux EINFO_LOG=/etc/init.d/samba
> MULTIOSDIRS=../lib64:../lib32 CONFIG_PROTECT_MASK=/etc/gentoo-release
> /etc/sandbox.d /etc/fonts/fonts.conf /etc/terminfo
> /etc/ca-certificates.conf /etc/revdep-rebuild RC_SVCNAME=samba PWD=/
> DISTCC_LOG= LANG=en_US.utf8 DISTCC_VERBOSE= DCCC_PATH=/usr/lib64/distcc/bin
> SVCNAME=samba SHLVL=1 HOME=/ RC_SERVICE=/etc/init.d/samba LESS=-R -M
> --shift 5 GCC_SPECS=
> ROOTPATH=/opt/bin:/usr/x86_64-pc-linux-gnu/gcc-bin/4.7.3 LESSOPEN=|lesspipe
> %s
>
> INFOPATH=/usr/share/info:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/info:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/info
> CONFIG_PROTECT=/var/bind _=/sbin/start-stop-daemon USER=root
>
> PATH=/bin:/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
> root 4449 0.0 0.8 549804 34704 ? S Sep20 1:47 \_
> /usr/sbin/samba
>
> MANPATH=/usr/local/share/man:/usr/share/man:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/man:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/man
> CONSOLE=/dev/console SHELL=/bin/sh TERM=linux EINFO_LOG=/etc/init.d/samba
> MULTIOSDIRS=../lib64:../lib32 CONFIG_PROTECT_MASK=/etc/gentoo-release
> /etc/sandbox.d /etc/fonts/fonts.conf /etc/terminfo
> /etc/ca-certificates.conf /etc/revdep-rebuild RC_SVCNAME=samba PWD=/
> DISTCC_LOG= LANG=en_US.utf8 DISTCC_VERBOSE= DCCC_PATH=/usr/lib64/distcc/bin
> SVCNAME=samba SHLVL=1 HOME=/ RC_SERVICE=/etc/init.d/samba LESS=-R -M
> --shift 5 GCC_SPECS=
> ROOTPATH=/opt/bin:/usr/x86_64-pc-linux-gnu/gcc-bin/4.7.3 LESSOPEN=|lesspipe
> %s
>
> INFOPATH=/usr/share/info:/usr/share/gcc-data/x86_64-pc-linux-gnu/4.7.3/info:/usr/share/binutils-data/x86_64-pc-linux-gnu/2.23.2/info
> CONFIG_PROTECT=/var/bind _=/sbin/start-stop-daemon USER=root
>
> PATH=/bin:/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
>
>
>
>
> /var/log/dmesg
>
> [584564.942178] samba_spnupdate invoked oom-killer: gfp_mask=0x201da,
> order=0, oom_score_adj=0
> [584564.942184] samba_spnupdate cpuset=/ mems_allowed=0
> [584564.942190] CPU: 3 PID: 7259 Comm: samba_spnupdate Not tainted
> 3.14.14-gentoo #1
> [584564.942192] Hardware name: VMware, Inc. VMware Virtual Platform/440BX
> Desktop Reference Platform, BIOS 6.00 07/30/2013
> [584564.942195] ffff880137861690 ffff88000202b998 ffffffff817124a8
> 0000000000000006
> [584564.942200] ffff8801378610d0 ffff88000202ba08 ffffffff8170ffc2
> ffff8801378610d0
> [584564.942212] 0000000000000202 ffffffff819c8390 0000000000000202
> ffff88000202ba08
> [584564.942223] Call Trace:
> [584564.942237] [<ffffffff817124a8>] dump_stack+0x46/0x58
> [584564.942244] [<ffffffff8170ffc2>] dump_header.isra.12+0x71/0x19c
> [584564.942256] [<ffffffff814202bf>] ? ___ratelimit+0xcb/0xe8
> [584564.942264] [<ffffffff8114e828>] oom_kill_process+0x68/0x31b
> [584564.942275] [<ffffffff810e85d8>] ? has_capability_noaudit+0x12/0x16
> [584564.942280] [<ffffffff8114e6e3>] ? oom_badness+0xb9/0x103
> [584564.942288] [<ffffffff8114eeb1>] out_of_memory+0x261/0x294
> [584564.942294] [<ffffffff811524ff>] __alloc_pages_nodemask+0x740/0x8a8
> [584564.942303] [<ffffffff8114d763>] filemap_fault+0x25a/0x373
> [584564.942316] [<ffffffff8116512b>] __do_fault+0x9f/0x3eb
> [584564.942328] [<ffffffff81168d62>] handle_mm_fault+0x420/0x88c
> [584564.942334] [<ffffffff8105d25f>] __do_page_fault+0x3cb/0x42a
> [584564.942338] [<ffffffff8116c577>] ? vma_merge+0x14f/0x227
> [584564.942343] [<ffffffff8116d3df>] ? do_brk+0x21a/0x243
> [584564.942347] [<ffffffff8105d2ff>] do_page_fault+0x22/0x27
> [584564.942352] [<ffffffff81718e08>] page_fault+0x28/0x30
> [584564.942354] Mem-Info:
> [584564.942356] DMA per-cpu:
> [584564.942359] CPU 0: hi: 0, btch: 1 usd: 0
> [584564.942361] CPU 1: hi: 0, btch: 1 usd: 0
> [584564.942363] CPU 2: hi: 0, btch: 1 usd: 0
> [584564.942365] CPU 3: hi: 0, btch: 1 usd: 0
> [584564.942367] CPU 4: hi: 0, btch: 1 usd: 0
> [584564.942368] CPU 5: hi: 0, btch: 1 usd: 0
> [584564.942370] CPU 6: hi: 0, btch: 1 usd: 0
> [584564.942372] CPU 7: hi: 0, btch: 1 usd: 0
> [584564.942374] DMA32 per-cpu:
> [584564.942378] CPU 0: hi: 186, btch: 31 usd: 0
> [584564.942385] CPU 1: hi: 186, btch: 31 usd: 0
> [584564.942387] CPU 2: hi: 186, btch: 31 usd: 0
> [584564.942391] CPU 3: hi: 186, btch: 31 usd: 153
> [584564.942397] CPU 4: hi: 186, btch: 31 usd: 0
> [584564.942399] CPU 5: hi: 186, btch: 31 usd: 0
> [584564.942401] CPU 6: hi: 186, btch: 31 usd: 0
> [584564.942403] CPU 7: hi: 186, btch: 31 usd: 0
> [584564.942404] Normal per-cpu:
> [584564.942406] CPU 0: hi: 186, btch: 31 usd: 0
> [584564.942408] CPU 1: hi: 186, btch: 31 usd: 0
> [584564.942410] CPU 2: hi: 186, btch: 31 usd: 0
> [584564.942412] CPU 3: hi: 186, btch: 31 usd: 87
> [584564.942414] CPU 4: hi: 186, btch: 31 usd: 0
> [584564.942416] CPU 5: hi: 186, btch: 31 usd: 0
> [584564.942418] CPU 6: hi: 186, btch: 31 usd: 0
> [584564.942420] CPU 7: hi: 186, btch: 31 usd: 0
> [584564.942427] active_anon:709038 inactive_anon:265964 isolated_anon:0
> active_file:39 inactive_file:0 isolated_file:0
> unevictable:0 dirty:0 writeback:0 unstable:0
> free:6876 slab_reclaimable:1421 slab_unreclaimable:6051
> mapped:0 shmem:51 pagetables:7629 bounce:0
> free_cma:0
> [584564.942440] DMA free:15764kB min:28kB low:32kB high:40kB
> active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB
> unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB
> managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB
> slab_reclaimable:0kB slab_unreclaimable:44kB kernel_stack:0kB
> pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB
> pages_scanned:0 all_unreclaimable? yes
> [584564.942442] lowmem_reserve[]: 0 2980 3940 3940
> [584564.942456] DMA32 free:9868kB min:6060kB low:7572kB high:9088kB
> active_anon:2367984kB inactive_anon:592000kB active_file:96kB
> inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB
> present:3129280kB managed:3052456kB mlocked:0kB dirty:0kB writeback:0kB
> mapped:0kB shmem:160kB slab_reclaimable:3736kB slab_unreclaimable:12268kB
> kernel_stack:792kB pagetables:21332kB unstable:0kB bounce:0kB free_cma:0kB
> writeback_tmp:0kB pages_scanned:268 all_unreclaimable? yes
> [584564.942459] lowmem_reserve[]: 0 0 960 960
> [584564.942472] Normal free:1872kB min:1952kB low:2440kB high:2928kB
> active_anon:468168kB inactive_anon:471856kB active_file:60kB
> inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB
> present:1048576kB managed:983676kB mlocked:0kB dirty:0kB writeback:0kB
> mapped:0kB shmem:44kB slab_reclaimable:1948kB slab_unreclaimable:11892kB
> kernel_stack:640kB pagetables:9184kB unstable:0kB bounce:0kB free_cma:0kB
> writeback_tmp:0kB pages_scanned:172 all_unreclaimable? yes
> [584564.942474] lowmem_reserve[]: 0 0 0 0
> [584564.942478] DMA: 1*4kB (U) 2*8kB (U) 0*16kB 0*32kB 0*64kB 1*128kB (U)
> 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (U) 3*4096kB (MR) = 15764kB
> [584564.942496] DMA32: 580*4kB (UEM) 16*8kB (M) 1*16kB (R) 2*32kB (R)
> 0*64kB 0*128kB 2*256kB (R) 0*512kB 1*1024kB (R) 1*2048kB (R) 1*4096kB (R) =
> 10208kB
> [584564.942515] Normal: 522*4kB (UE) 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB
> 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 2088kB
> [584564.942528] 7944 total pagecache pages
> [584564.942530] 7751 pages in swap cache
> [584564.942533] Swap cache stats: add 722854, delete 715103, find
> 2187050/2296416
> [584564.942534] Free swap = 0kB
> [584564.942536] Total swap = 1953788kB
> [584564.942537] 1048462 pages RAM
> [584564.942538] 0 pages HighMem/MovableOnly
> [584564.942540] 16225 pages reserved
> [584564.942541] [ pid ] uid tgid total_vm rss nr_ptes swapents
> oom_score_adj name
> [584564.942566] [ 2893] 0 2893 9086 1 20 120
> -1000 systemd-udevd
> [584564.942571] [ 3410] 0 3410 24049 0 13 75
> 0 vmware-vmblock-
> [584564.942574] [ 3418] 0 3418 15216 140 33 362
> 0 vmtoolsd
> [584564.942578] [ 4208] 0 4208 6814 9 17 53
> 0 syslog-ng
> [584564.942581] [ 4209] 0 4209 121432 165 43 297
> 0 syslog-ng
> [584564.942584] [ 4239] 0 4239 18499 56 38 199
> 0 cupsd
> [584564.942587] [ 4263] 40 4263 263053 4610 243 15400
> 0 named
> [584564.942591] [ 4308] 102 4308 18642 2737 37 7041
> 0 dhcpd
> [584564.942594] [ 4332] 0 4332 17778 1 35 185
> -1000 sshd
> [584564.942598] [ 4510] 0 4510 6185 33 15 104
> 0 ntpd
> [584564.942601] [ 4530] 0 4530 136601 955 259 7167
> 0 samba
> [584564.942604] [ 4552] 0 4552 3677 1 12 38
> 0 agetty
> [584564.942607] [ 4553] 0 4553 3677 1 11 40
> 0 agetty
> [584564.942610] [ 4554] 0 4554 3677 1 12 39
> 0 agetty
> [584564.942613] [ 4555] 0 4555 3677 1 12 40
> 0 agetty
> [584564.942616] [ 4556] 0 4556 3677 1 12 39
> 0 agetty
> [584564.942619] [ 4557] 0 4557 3677 1 12 39
> 0 agetty
> [584564.942622] [ 4561] 0 4561 136601 941 234 7180
> 0 samba
> [584564.942625] [ 4562] 0 4562 118865 371 227 7614
> 0 smbd
> [584564.942629] [ 4563] 0 4563 137807 2943 266 5184
> 0 samba
> [584564.942632] [ 4564] 0 4564 137633 1419 259 6706
> 0 samba
> [584564.942635] [ 4565] 0 4565 136601 951 234 7170
> 0 samba
> [584564.942638] [ 4566] 0 4566 137738 3458 264 4670
> 0 samba
> [584564.942640] [ 4567] 0 4567 136595 1714 252 6407
> 0 samba
> [584564.942643] [ 4568] 0 4568 137639 2329 262 5798
> 0 samba
> [584564.942647] [ 4569] 0 4569 1502007 924883 2926 445966
> 0 samba
> [584564.942650] [ 4570] 0 4570 137445 1884 246 6237
> 0 samba
> [584564.942653] [ 4571] 0 4571 136601 955 234 7166
> 0 samba
> [584564.942656] [ 4572] 0 4572 137633 2748 260 5377
> 0 samba
> [584564.942659] [ 4573] 0 4573 136595 1835 244 6286
> 0 samba
> [584564.942662] [ 4578] 0 4578 118865 405 201 7564
> 0 smbd
> [584564.942665] [ 7257] 3000064 7257 127698 2083 245 5962
> 0 smbd
> [584564.942668] [ 7258] 0 7258 131156 10549 245 0
> 0 samba_dnsupdate
> [584564.942671] [ 7259] 0 7259 122014 10015 234 0
> 0 samba_spnupdate
> [584564.942675] Out of memory: Kill process 4569 (samba) score 887 or
> sacrifice child
> [584564.942678] Killed process 4569 (samba) total-vm:6008028kB,
> anon-rss:3699524kB, file-rss:8kB
>
More information about the samba-technical
mailing list