[SCM] Samba Shared Repository - branch master updated

Jeremy Allison jra at samba.org
Thu Feb 9 22:59:02 UTC 2017


The branch, master has been updated
       via  a592bdc docs: Add missing spaces in man smb.conf.
       via  6c3aacc s3-vfs: Only walk the directory once in open_and_sort_dir()
       via  dd25d75 Move pthreadpool to top of the tree.
       via  3e2a367 wafsamba: Remove 2010 comments that seems not accurate anymore
       via  468301b wafsamba: Move command line option function labelled as 'samba3' to the common set of functions
      from  45df61e Switch on the sortedLinks Flag on new databases

https://git.samba.org/?p=samba.git;a=shortlog;h=master


- Log -----------------------------------------------------------------
commit a592bdc51c47db7274f92031fe192b82d552ca05
Author: Karolin Seeger <kseeger at samba.org>
Date:   Thu Feb 9 11:27:45 2017 +0100

    docs: Add missing spaces in man smb.conf.
    
    Signed-off-by: Karolin Seeger <kseeger at samba.org>
    Reviewed-by: Jeremy Allison <jra at samba.org>
    
    Autobuild-User(master): Jeremy Allison <jra at samba.org>
    Autobuild-Date(master): Thu Feb  9 23:58:02 CET 2017 on sn-devel-144

commit 6c3aaccd0eb648e31fd2717aaca0187966e125d5
Author: Andreas Schneider <asn at samba.org>
Date:   Thu Feb 9 15:05:01 2017 +0100

    s3-vfs: Only walk the directory once in open_and_sort_dir()
    
    On a slow filesystem or network filesystem this can make a huge
    difference.
    
    BUG: https://bugzilla.samba.org/show_bug.cgi?id=12571
    
    Signed-off-by: Andreas Schneider <asn at samba.org>
    Reviewed-by: Jeremy Allison <jra at samba.org>

commit dd25d75b96e9930e441663e5d8e95a84eeed5c62
Author: Matthieu Patou <mat at matws.net>
Date:   Fri Feb 3 15:13:49 2017 -0800

    Move pthreadpool to top of the tree.
    
    Signed-off-by: Matthieu Patou <mat at matws.net>
    Reviewed-by: Jeremy Allison <jra at samba.org>

commit 3e2a36774f44cf711611fcd3cd3c2c66b9efd662
Author: Matthieu Patou <mat at matws.net>
Date:   Wed Feb 8 12:01:50 2017 -0800

    wafsamba: Remove 2010 comments that seems not accurate anymore
    
    In my tests default value is correctly used and if we provide explicitly
    a --with it will comply with the store_true and if we provide --without
    then it will comply with the store_false
    
    Change-Id: I820a7f2f08c51ec23b694bce7009c3891d4ab8ef
    Reviewed-by: Jeremy Allison <jra at samba.org>

commit 468301bedda32057c12a2007f5500205ff8b0252
Author: Matthieu Patou <mat at matws.net>
Date:   Tue Feb 7 22:58:40 2017 -0800

    wafsamba: Move command line option function labelled as 'samba3' to the common set of functions
    
    It allows to be used for things that are not 'samba3' only (or more
    accurately things not in common and not related to the AD DC
    implementation)
    
    Signed-off-by: Matthieu Patou <mat at matws.net>
    Reviewed-by: Jeremy Allison <jra at samba.org>

-----------------------------------------------------------------------

Summary of changes:
 buildtools/wafsamba/samba3.py                      | 31 +----------
 buildtools/wafsamba/samba_utils.py                 | 25 +++++++++
 docs-xml/smbdotconf/ldap/ldapssl.xml               |  6 +--
 {source3/lib => lib}/pthreadpool/Makefile          |  0
 {source3/lib => lib}/pthreadpool/pthreadpool.c     |  0
 {source3/lib => lib}/pthreadpool/pthreadpool.h     |  0
 .../lib => lib}/pthreadpool/pthreadpool_pipe.c     |  0
 .../lib => lib}/pthreadpool/pthreadpool_pipe.h     |  0
 .../lib => lib}/pthreadpool/pthreadpool_sync.c     |  0
 .../lib => lib}/pthreadpool/pthreadpool_tevent.c   |  0
 .../lib => lib}/pthreadpool/pthreadpool_tevent.h   |  0
 {source3/lib => lib}/pthreadpool/tests.c           |  0
 {source3/lib => lib}/pthreadpool/wscript_build     |  6 +--
 source3/lib/fncall.c                               |  2 +-
 source3/modules/vfs_aio_pthread.c                  |  2 +-
 source3/modules/vfs_dirsort.c                      | 63 +++++++++++++---------
 source3/torture/bench_pthreadpool.c                |  2 +-
 source3/wscript                                    |  8 ---
 wscript                                            | 10 ++++
 wscript_build                                      |  1 +
 20 files changed, 86 insertions(+), 70 deletions(-)
 rename {source3/lib => lib}/pthreadpool/Makefile (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool.c (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool.h (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool_pipe.c (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool_pipe.h (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool_sync.c (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool_tevent.c (100%)
 rename {source3/lib => lib}/pthreadpool/pthreadpool_tevent.h (100%)
 rename {source3/lib => lib}/pthreadpool/tests.c (100%)
 rename {source3/lib => lib}/pthreadpool/wscript_build (86%)


Changeset truncated at 500 lines:

diff --git a/buildtools/wafsamba/samba3.py b/buildtools/wafsamba/samba3.py
index 6d06fd9..44daff9 100644
--- a/buildtools/wafsamba/samba3.py
+++ b/buildtools/wafsamba/samba3.py
@@ -2,37 +2,10 @@
 # and for SAMBA_ macros for building libraries, binaries etc
 
 import Options, Build, os
-from optparse import SUPPRESS_HELP
-from samba_utils import os_path_relpath, TO_LIST
+from samba_utils import os_path_relpath, TO_LIST, samba_add_onoff_option
 from samba_autoconf import library_flags
 
-
-def SAMBA3_ADD_OPTION(opt, option, help=(), dest=None, default=True,
-                      with_name="with", without_name="without"):
-    if default is None:
-        default_str = "auto"
-    elif default is True:
-        default_str = "yes"
-    elif default is False:
-        default_str = "no"
-    else:
-        default_str = str(default)
-
-    if help == ():
-        help = ("Build with %s support (default=%s)" % (option, default_str))
-    if dest is None:
-        dest = "with_%s" % option.replace('-', '_')
-
-    with_val = "--%s-%s" % (with_name, option)
-    without_val = "--%s-%s" % (without_name, option)
-
-    #FIXME: This is broken and will always default to "default" no matter if
-    # --with or --without is chosen.
-    opt.add_option(with_val, help=help, action="store_true", dest=dest,
-                   default=default)
-    opt.add_option(without_val, help=SUPPRESS_HELP, action="store_false",
-                   dest=dest)
-Options.Handler.SAMBA3_ADD_OPTION = SAMBA3_ADD_OPTION
+Options.Handler.SAMBA3_ADD_OPTION = samba_add_onoff_option
 
 def SAMBA3_IS_STATIC_MODULE(bld, module):
     '''Check whether module is in static list'''
diff --git a/buildtools/wafsamba/samba_utils.py b/buildtools/wafsamba/samba_utils.py
index 49a8759..0f95c12 100644
--- a/buildtools/wafsamba/samba_utils.py
+++ b/buildtools/wafsamba/samba_utils.py
@@ -2,6 +2,7 @@
 # and for SAMBA_ macros for building libraries, binaries etc
 
 import os, sys, re, fnmatch, shlex
+from optparse import SUPPRESS_HELP
 import Build, Options, Utils, Task, Logs, Configure
 from TaskGen import feature, before, after
 from Configure import conf, ConfigurationContext
@@ -669,3 +670,27 @@ def samba_before_apply_obj_vars(self):
         if is_standard_libpath(v, i):
             v['LIBPATH'].remove(i)
 
+def samba_add_onoff_option(opt, option, help=(), dest=None, default=True,
+                           with_name="with", without_name="without"):
+    if default is None:
+        default_str = "auto"
+    elif default is True:
+        default_str = "yes"
+    elif default is False:
+        default_str = "no"
+    else:
+        default_str = str(default)
+
+    if help == ():
+        help = ("Build with %s support (default=%s)" % (option, default_str))
+    if dest is None:
+        dest = "with_%s" % option.replace('-', '_')
+
+    with_val = "--%s-%s" % (with_name, option)
+    without_val = "--%s-%s" % (without_name, option)
+
+    opt.add_option(with_val, help=help, action="store_true", dest=dest,
+                   default=default)
+    opt.add_option(without_val, help=SUPPRESS_HELP, action="store_false",
+                   dest=dest)
+Options.Handler.samba_add_onoff_option = samba_add_onoff_option
diff --git a/docs-xml/smbdotconf/ldap/ldapssl.xml b/docs-xml/smbdotconf/ldap/ldapssl.xml
index a53d726..eba9193 100644
--- a/docs-xml/smbdotconf/ldap/ldapssl.xml
+++ b/docs-xml/smbdotconf/ldap/ldapssl.xml
@@ -14,7 +14,7 @@
 
 	<para>LDAP connections should be secured where possible. This may be
 	done setting <emphasis>either</emphasis> this parameter to
-	<parameter moreinfo="none">start tls</parameter>
+	<parameter moreinfo="none">start tls</parameter> 
 	<emphasis>or</emphasis> by specifying <parameter moreinfo="none">ldaps://</parameter> in
         the URL argument of <smbconfoption name="passdb backend"/>.</para>
 
@@ -34,9 +34,9 @@
 	</itemizedlist>
 	<para>
 	Please note that this parameter does only affect <emphasis>rpc</emphasis>
-	methods. To enable the LDAPv3 StartTLS extended operation (RFC2830) for
+	methods. To enable the LDAPv3 StartTLS extended operation (RFC2830) for 
 	<emphasis>ads</emphasis>, set
-	<smbconfoption name="ldap ssl">start tls</smbconfoption>
+	<smbconfoption name="ldap ssl">start tls</smbconfoption> 
 	<emphasis>and</emphasis>
 	<smbconfoption name="ldap ssl ads">yes</smbconfoption>.
 	See <refentrytitle>smb.conf</refentrytitle><manvolnum>5</manvolnum>
diff --git a/source3/lib/pthreadpool/Makefile b/lib/pthreadpool/Makefile
similarity index 100%
rename from source3/lib/pthreadpool/Makefile
rename to lib/pthreadpool/Makefile
diff --git a/source3/lib/pthreadpool/pthreadpool.c b/lib/pthreadpool/pthreadpool.c
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool.c
rename to lib/pthreadpool/pthreadpool.c
diff --git a/source3/lib/pthreadpool/pthreadpool.h b/lib/pthreadpool/pthreadpool.h
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool.h
rename to lib/pthreadpool/pthreadpool.h
diff --git a/source3/lib/pthreadpool/pthreadpool_pipe.c b/lib/pthreadpool/pthreadpool_pipe.c
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool_pipe.c
rename to lib/pthreadpool/pthreadpool_pipe.c
diff --git a/source3/lib/pthreadpool/pthreadpool_pipe.h b/lib/pthreadpool/pthreadpool_pipe.h
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool_pipe.h
rename to lib/pthreadpool/pthreadpool_pipe.h
diff --git a/source3/lib/pthreadpool/pthreadpool_sync.c b/lib/pthreadpool/pthreadpool_sync.c
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool_sync.c
rename to lib/pthreadpool/pthreadpool_sync.c
diff --git a/source3/lib/pthreadpool/pthreadpool_tevent.c b/lib/pthreadpool/pthreadpool_tevent.c
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool_tevent.c
rename to lib/pthreadpool/pthreadpool_tevent.c
diff --git a/source3/lib/pthreadpool/pthreadpool_tevent.h b/lib/pthreadpool/pthreadpool_tevent.h
similarity index 100%
rename from source3/lib/pthreadpool/pthreadpool_tevent.h
rename to lib/pthreadpool/pthreadpool_tevent.h
diff --git a/source3/lib/pthreadpool/tests.c b/lib/pthreadpool/tests.c
similarity index 100%
rename from source3/lib/pthreadpool/tests.c
rename to lib/pthreadpool/tests.c
diff --git a/source3/lib/pthreadpool/wscript_build b/lib/pthreadpool/wscript_build
similarity index 86%
rename from source3/lib/pthreadpool/wscript_build
rename to lib/pthreadpool/wscript_build
index 8195af7..d530463 100644
--- a/source3/lib/pthreadpool/wscript_build
+++ b/lib/pthreadpool/wscript_build
@@ -1,14 +1,14 @@
 #!/usr/bin/env python
 
 if bld.env.WITH_PTHREADPOOL:
-    bld.SAMBA3_SUBSYSTEM('PTHREADPOOL',
+    bld.SAMBA_SUBSYSTEM('PTHREADPOOL',
                          source='''pthreadpool.c
                                    pthreadpool_pipe.c
                                    pthreadpool_tevent.c
                                 ''',
                          deps='pthread rt replace tevent-util')
 else:
-    bld.SAMBA3_SUBSYSTEM('PTHREADPOOL',
+    bld.SAMBA_SUBSYSTEM('PTHREADPOOL',
                          source='''pthreadpool_sync.c
                                    pthreadpool_pipe.c
                                    pthreadpool_tevent.c
@@ -16,7 +16,7 @@ else:
                          deps='replace tevent-util')
 
 
-bld.SAMBA3_BINARY('pthreadpooltest',
+bld.SAMBA_BINARY('pthreadpooltest',
                   source='tests.c',
                   deps='PTHREADPOOL',
                   enabled=bld.env.WITH_PTHREADPOOL,
diff --git a/source3/lib/fncall.c b/source3/lib/fncall.c
index 0923c14..34db472 100644
--- a/source3/lib/fncall.c
+++ b/source3/lib/fncall.c
@@ -20,7 +20,7 @@
 #include "includes.h"
 #include "../lib/util/tevent_unix.h"
 
-#include "lib/pthreadpool/pthreadpool_pipe.h"
+#include "../lib/pthreadpool/pthreadpool_pipe.h"
 
 struct fncall_state {
 	struct fncall_context *ctx;
diff --git a/source3/modules/vfs_aio_pthread.c b/source3/modules/vfs_aio_pthread.c
index 6edf250..97ae86f 100644
--- a/source3/modules/vfs_aio_pthread.c
+++ b/source3/modules/vfs_aio_pthread.c
@@ -26,7 +26,7 @@
 #include "system/shmem.h"
 #include "smbd/smbd.h"
 #include "smbd/globals.h"
-#include "lib/pthreadpool/pthreadpool_pipe.h"
+#include "../lib/pthreadpool/pthreadpool_pipe.h"
 #ifdef HAVE_LINUX_FALLOC_H
 #include <linux/falloc.h>
 #endif
diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c
index 4a3e152..cd5597a 100644
--- a/source3/modules/vfs_dirsort.c
+++ b/source3/modules/vfs_dirsort.c
@@ -65,8 +65,10 @@ static bool get_sorted_dir_mtime(vfs_handle_struct *handle,
 static bool open_and_sort_dir(vfs_handle_struct *handle,
 				struct dirsort_privates *data)
 {
-	unsigned int i = 0;
-	unsigned int total_count = 0;
+	uint32_t total_count = 0;
+	/* This should be enough for most use cases */
+	uint32_t dirent_allocated = 64;
+	struct dirent *dp;
 
 	data->number_of_entries = 0;
 
@@ -74,38 +76,51 @@ static bool open_and_sort_dir(vfs_handle_struct *handle,
 		return false;
 	}
 
-	while (SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL)
-	       != NULL) {
-		total_count++;
-	}
-
-	if (total_count == 0) {
+	dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
+	if (dp == NULL) {
 		return false;
 	}
 
-	/* Open the underlying directory and count the number of entries
-	   Skip back to the beginning as we'll read it again */
-	SMB_VFS_NEXT_REWINDDIR(handle, data->source_directory);
-
 	/* Set up an array and read the directory entries into it */
 	TALLOC_FREE(data->directory_list); /* destroy previous cache if needed */
 	data->directory_list = talloc_zero_array(data,
-					struct dirent,
-					total_count);
-	if (!data->directory_list) {
+						 struct dirent,
+						 dirent_allocated);
+	if (data->directory_list == NULL) {
 		return false;
 	}
-	for (i = 0; i < total_count; i++) {
-		struct dirent *dp = SMB_VFS_NEXT_READDIR(handle,
-						data->source_directory,
-						NULL);
-		if (dp == NULL) {
-			break;
+
+	do {
+		if (total_count >= dirent_allocated) {
+			struct dirent *dlist;
+
+			/*
+			 * Be memory friendly.
+			 *
+			 * We should not double the amount of memory. With a lot
+			 * of files we reach easily 50MB, and doubling will
+			 * get much bigger just for a few files more.
+			 *
+			 * For 200k files this means 50 memory reallocations.
+			 */
+			dirent_allocated += 4096;
+
+			dlist = talloc_realloc(data,
+					       data->directory_list,
+					       struct dirent,
+					       dirent_allocated);
+			if (dlist == NULL) {
+				break;
+			}
+			data->directory_list = dlist;
 		}
-		data->directory_list[i] = *dp;
-	}
+		data->directory_list[total_count] = *dp;
+
+		total_count++;
+		dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
+	} while (dp != NULL);
 
-	data->number_of_entries = i;
+	data->number_of_entries = total_count;
 
 	/* Sort the directory entries by name */
 	TYPESAFE_QSORT(data->directory_list, data->number_of_entries, compare_dirent);
diff --git a/source3/torture/bench_pthreadpool.c b/source3/torture/bench_pthreadpool.c
index 82a84cf..4269b53 100644
--- a/source3/torture/bench_pthreadpool.c
+++ b/source3/torture/bench_pthreadpool.c
@@ -19,7 +19,7 @@
  */
 
 #include "includes.h"
-#include "lib/pthreadpool/pthreadpool_pipe.h"
+#include "../lib/pthreadpool/pthreadpool_pipe.h"
 #include "proto.h"
 
 extern int torture_numops;
diff --git a/source3/wscript b/source3/wscript
index 443affd..821d4ed 100644
--- a/source3/wscript
+++ b/source3/wscript
@@ -46,7 +46,6 @@ def set_options(opt):
     opt.SAMBA3_ADD_OPTION('quotas')
     opt.SAMBA3_ADD_OPTION('sendfile-support')
     opt.SAMBA3_ADD_OPTION('utmp')
-    opt.SAMBA3_ADD_OPTION('pthreadpool', with_name="enable", without_name="disable", default=True)
     opt.SAMBA3_ADD_OPTION('avahi', with_name="enable", without_name="disable")
     opt.SAMBA3_ADD_OPTION('iconv')
     opt.SAMBA3_ADD_OPTION('acl-support')
@@ -1564,13 +1563,6 @@ main() {
         conf.DEFINE('WITH_PROFILE', 1);
         conf.CHECK_FUNCS('getrusage', headers="sys/time.h sys/resource.h")
 
-    if Options.options.with_pthreadpool:
-        if conf.CONFIG_SET('HAVE_PTHREAD'):
-            conf.DEFINE('WITH_PTHREADPOOL', '1')
-        else:
-            Logs.warn("pthreadpool support cannot be enabled when pthread support was not found")
-            conf.undefine('WITH_PTHREADPOOL')
-
     if (conf.CHECK_HEADERS('linux/ioctl.h sys/ioctl.h linux/fs.h') and
         conf.CHECK_DECLS('FS_IOC_GETFLAGS FS_COMPR_FL', headers='linux/fs.h')):
             conf.DEFINE('HAVE_LINUX_IOCTL', '1')
diff --git a/wscript b/wscript
index 9168db1..9706e0f 100644
--- a/wscript
+++ b/wscript
@@ -9,6 +9,7 @@ VERSION=None
 import sys, os, tempfile
 sys.path.insert(0, srcdir+"/buildtools/wafsamba")
 import wafsamba, Options, samba_dist, samba_git, Scripting, Utils, samba_version
+import Logs, samba_utils
 
 
 samba_dist.DIST_DIRS('.')
@@ -44,6 +45,7 @@ def set_options(opt):
     opt.RECURSE('source3')
     opt.RECURSE('lib/util')
     opt.RECURSE('ctdb')
+    opt.samba_add_onoff_option('pthreadpool', with_name="enable", without_name="disable", default=True)
 
     opt.add_option('--with-system-mitkrb5',
                    help='enable system MIT krb5 build (includes Samba 4 client and Samba 3 code base).'+
@@ -183,6 +185,14 @@ def configure(conf):
         if Options.options.with_system_mitkrb5:
             raise Utils.WafError('--with-ntvfs-fileserver conflicts with --with-system-mitkrb5')
         conf.DEFINE('WITH_NTVFS_FILESERVER', 1)
+
+    if Options.options.with_pthreadpool:
+        if conf.CONFIG_SET('HAVE_PTHREAD'):
+            conf.DEFINE('WITH_PTHREADPOOL', '1')
+        else:
+            Logs.warn("pthreadpool support cannot be enabled when pthread support was not found")
+            conf.undefine('WITH_PTHREADPOOL')
+
     conf.RECURSE('source3')
     conf.RECURSE('lib/texpect')
     if conf.env.with_ctdb:
diff --git a/wscript_build b/wscript_build
index 0c3a2ae..954eed1 100644
--- a/wscript_build
+++ b/wscript_build
@@ -145,6 +145,7 @@ bld.RECURSE('source4/scripting')
 bld.RECURSE('pidl')
 bld.RECURSE('lib')
 bld.RECURSE('libds/common')
+bld.RECURSE('lib/pthreadpool')
 bld.RECURSE('source3')
 bld.RECURSE('dfs_server')
 bld.RECURSE('file_server')


-- 
Samba Shared Repository



More information about the samba-cvs mailing list